Users must make sure that no in-progress commits on the tables of source catalog during
+ * registration.
+ *
+ * @param identifier table identifier to register or migrate
+ * @return {@code this} for use in a chained invocation
+ */
+ public CatalogMigrator registerTable(TableIdentifier identifier) {
+ Preconditions.checkArgument(identifier != null, "Identifier is null");
+
+ boolean isRegistered = registerTableToTargetCatalog(identifier);
+ if (isRegistered) {
+ resultBuilder.addRegisteredTableIdentifiers(identifier);
+ } else {
+ resultBuilder.addFailedToRegisterTableIdentifiers(identifier);
+ }
+
+ try {
+ if (isRegistered
+ && deleteEntriesFromSourceCatalog()
+ && !sourceCatalog().dropTable(identifier, false)) {
+ resultBuilder.addFailedToDeleteTableIdentifiers(identifier);
+ }
+ } catch (Exception exception) {
+ resultBuilder.addFailedToDeleteTableIdentifiers(identifier);
+ if (enableStacktrace()) {
+ LOG.error("Failed to delete the table after migration {}", identifier, exception);
+ } else {
+ LOG.error(
+ "Failed to delete the table after migration {} : {}",
+ identifier,
+ exception.getMessage());
+ }
+ }
+ return this;
+ }
+
+ public CatalogMigrationResult result() {
+ processedNamespaces.clear();
+ return resultBuilder.build();
+ }
+
+ protected void createNamespacesIfNotExistOnTargetCatalog(Namespace identifierNamespace) {
+ if (!processedNamespaces.contains(identifierNamespace)) {
+ String[] levels = identifierNamespace.levels();
+ for (int index = 0; index < levels.length; index++) {
+ Namespace namespace = Namespace.of(Arrays.copyOfRange(levels, 0, index + 1));
+ if (processedNamespaces.add(namespace)) {
+ try {
+ ((SupportsNamespaces) targetCatalog()).createNamespace(namespace);
+ } catch (AlreadyExistsException ex) {
+ LOG.debug(
+ "{}.Ignoring the error as forcefully creating the namespace even if it exists to avoid "
+ + "namespaceExists() check.",
+ ex.getMessage());
+ }
+ }
+ }
+ }
+ }
+
+ protected void getAllNamespacesFromSourceCatalog(Namespace namespace, Set visited) {
+ if (!visited.add(namespace)) {
+ return;
+ }
+ List children = ((SupportsNamespaces) sourceCatalog()).listNamespaces(namespace);
+ for (Namespace child : children) {
+ getAllNamespacesFromSourceCatalog(child, visited);
+ }
+ }
+
+ private boolean registerTableToTargetCatalog(TableIdentifier tableIdentifier) {
+ try {
+ createNamespacesIfNotExistOnTargetCatalog(tableIdentifier.namespace());
+ // register the table to the target catalog
+ TableOperations ops = ((BaseTable) sourceCatalog().loadTable(tableIdentifier)).operations();
+ targetCatalog().registerTable(tableIdentifier, ops.current().metadataFileLocation());
+ LOG.info("Successfully registered the table {}", tableIdentifier);
+ return true;
+ } catch (Exception ex) {
+ if (enableStacktrace()) {
+ LOG.error("Unable to register the table {}", tableIdentifier, ex);
+ } else {
+ LOG.error("Unable to register the table {} : {}", tableIdentifier, ex.getMessage());
+ }
+ return false;
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
new file mode 100644
index 00000000..c41b5253
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.IntStream;
+import nl.altindag.log.LogCaptor;
+import nl.altindag.log.model.LogEvent;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public abstract class AbstractTestCatalogMigrator extends AbstractTest {
+
+ protected static final Namespace NS1 = Namespace.of("ns1");
+ protected static final Namespace NS2 = Namespace.of("ns2");
+ protected static final Namespace NS3 = Namespace.of("ns3");
+ protected static final Namespace NS1_NS2 = Namespace.of("ns1", "ns2");
+ protected static final Namespace NS1_NS3 = Namespace.of("ns1", "ns3");
+ protected static final Namespace NS1_NS2_NS3 = Namespace.of("ns1", "ns2", "ns3");
+
+ protected static final TableIdentifier TBL = TableIdentifier.parse("tblz");
+ protected static final TableIdentifier NS1_TBL = TableIdentifier.of(NS1, "tblz");
+ protected static final TableIdentifier NS2_TBL = TableIdentifier.of(NS2, "tblz");
+ protected static final TableIdentifier NS3_TBL = TableIdentifier.of(NS3, "tblz");
+ protected static final TableIdentifier NS1_NS2_TBL = TableIdentifier.of(NS1_NS2, "tblz");
+ protected static final TableIdentifier NS1_NS3_TBL = TableIdentifier.of(NS1_NS3, "tblz");
+ protected static final TableIdentifier NS1_NS2_NS3_TBL = TableIdentifier.of(NS1_NS2_NS3, "tblz");
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ dropTables();
+ }
+
+ protected static void initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(true, catalogType, additionalProp);
+ createNamespacesForSourceCatalog();
+ }
+
+ protected static void initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(false, catalogType, additionalProp);
+ createNamespacesForTargetCatalog();
+ }
+
+ private static void initializeCatalog(
+ boolean isSourceCatalog,
+ CatalogMigrationUtil.CatalogType catalogType,
+ Map additionalProp) {
+ Map properties;
+ switch (catalogType) {
+ case HADOOP:
+ properties = hadoopCatalogProperties(isSourceCatalog);
+ break;
+ case NESSIE:
+ properties = nessieCatalogProperties(isSourceCatalog);
+ break;
+ case HIVE:
+ properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported for catalog type: %s", catalogType));
+ }
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties,
+ catalogType,
+ isSourceCatalog ? "sourceCatalog" : "targetCatalog" + "_" + catalogType,
+ null,
+ null);
+ if (isSourceCatalog) {
+ sourceCatalog = catalog;
+ } else {
+ targetCatalog = catalog;
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegister(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ CatalogMigrationResult result = registerAllTables(deleteSourceTables);
+
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ if (deleteSourceTables) {
+ // table should be deleted after migration from source catalog
+ Assertions.assertThat(sourceCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(sourceCatalog.listTables(BAR)).isEmpty();
+ } else {
+ // tables should be present in source catalog.
+ Assertions.assertThat(sourceCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(sourceCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterSelectedTables(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // using `--identifiers` option
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(BAR_TBL3).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(BAR_TBL3);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+
+ // using --identifiers-regex option which matches all the tables starts with "foo."
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+ catalogMigrator
+ .getMatchingTableIdentifiers("^foo\\..*")
+ .forEach(catalogMigrator::registerTable);
+ result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterError(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // use invalid namespace which leads to NoSuchTableException
+ TableIdentifier identifier = TableIdentifier.parse("dummy.tbl3");
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(identifier).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(identifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // try to register same table twice which leads to AlreadyExistsException
+ result = catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ result = catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).contains(FOO_TBL2);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithFewFailures(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // register only foo.tbl2
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ if (deleteSourceTables) {
+ // create a table with the same name in source catalog which got deleted.
+ sourceCatalog.createTable(FOO_TBL2, schema);
+ }
+
+ // register all the tables from source catalog again. So that `foo.tbl2` will fail to register.
+ result = registerAllTables(deleteSourceTables);
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, BAR_TBL3, BAR_TBL4);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).contains(FOO_TBL2);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterLargeNumberOfTables(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ IntStream.range(0, 240)
+ .forEach(val -> sourceCatalog.createTable(TableIdentifier.of(FOO, "tblx" + val), schema));
+
+ CatalogMigrationResult result;
+ result = registerAllTables(deleteSourceTables);
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).hasSize(244);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).hasSize(242);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testListingTableIdentifiers(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+
+ // should list all the tables from all the namespace when regex is null.
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+
+ // list the tables whose identifier starts with "foo."
+ matchingTableIdentifiers = catalogMigrator.getMatchingTableIdentifiers("^foo\\..*");
+ Assertions.assertThat(matchingTableIdentifiers).containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+
+ // test filter that doesn't match any table.
+ matchingTableIdentifiers = catalogMigrator.getMatchingTableIdentifiers("^dev\\..*");
+ Assertions.assertThat(matchingTableIdentifiers).isEmpty();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithNewNamespace(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ TableIdentifier tbl5 = TableIdentifier.of(DB1, "tbl5");
+ // namespace "db1" exists only in source catalog
+ sourceCatalog.createTable(tbl5, schema);
+
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(tbl5).result();
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(tbl5);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(DB1)).containsExactly(tbl5);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
+ public void testStacktrace(boolean deleteSourceTables, boolean enableStacktrace) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ TableIdentifier identifier = TableIdentifier.parse("db.dummy_table");
+
+ ImmutableCatalogMigrator migrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(deleteSourceTables)
+ .enableStacktrace(enableStacktrace)
+ .build();
+ try (LogCaptor logCaptor = LogCaptor.forClass(CatalogMigrator.class)) {
+ CatalogMigrationResult result = migrator.registerTable(identifier).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(identifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(logCaptor.getLogEvents()).hasSize(1);
+ LogEvent logEvent = logCaptor.getLogEvents().get(0);
+ if (enableStacktrace) {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo("Unable to register the table db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable())
+ .isPresent()
+ .get()
+ .isInstanceOf(NoSuchTableException.class);
+ } else {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo(
+ "Unable to register the table db.dummy_table : Table does not exist: db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable()).isEmpty();
+ }
+ }
+ }
+
+ protected CatalogMigrator catalogMigratorWithDefaultArgs(boolean deleteSourceTables) {
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(deleteSourceTables)
+ .build();
+ }
+
+ private CatalogMigrationResult registerAllTables(boolean deleteSourceTables) {
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+ catalogMigrator.getMatchingTableIdentifiers(null).forEach(catalogMigrator::registerTable);
+ return catalogMigrator.result();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java
new file mode 100644
index 00000000..c4ed478b
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import static org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest.FOO_TBL1;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.nessie.NessieCatalog;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+public class CatalogMigrationUtilTest {
+
+ private static @TempDir Path logDir;
+
+ private static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", logDir.toAbsolutePath().toString());
+ }
+
+ static Stream blankOrNullStrings() {
+ return Stream.of("", " ", null);
+ }
+
+ @ParameterizedTest()
+ @MethodSource("blankOrNullStrings")
+ public void testCustomCatalogWithoutImpl(String impl) {
+ Assertions.assertThatThrownBy(
+ () ->
+ CatalogMigrationUtil.buildCatalog(
+ Collections.emptyMap(),
+ CatalogMigrationUtil.CatalogType.CUSTOM,
+ "catalogName",
+ impl,
+ Collections.emptyMap()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Need to specify the fully qualified class name of the custom catalog impl");
+ }
+
+ @Test
+ public void testInvalidArgs() {
+ Assertions.assertThatThrownBy(
+ () -> CatalogMigrationUtil.buildCatalog(null, null, null, null, null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("catalog properties is null");
+
+ Assertions.assertThatThrownBy(
+ () -> CatalogMigrationUtil.buildCatalog(Collections.emptyMap(), null, null, null, null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("catalog type is null");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ CatalogMigrationUtil.buildCatalog(
+ Collections.emptyMap(),
+ CatalogMigrationUtil.CatalogType.CUSTOM,
+ "catalogName",
+ "abc",
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Cannot initialize Catalog implementation abc: Cannot find constructor for interface");
+ }
+
+ @Test
+ public void testBuildHadoopCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("type", "hadoop");
+
+ Map conf = new HashMap<>();
+ conf.put("k1", "v1");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.HADOOP, "catalogName", null, conf);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(HadoopCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ Assertions.assertThat(((HadoopCatalog) catalog).getConf().get("k1")).isEqualTo("v1");
+ Schema schema =
+ new Schema(
+ Types.StructType.of(Types.NestedField.required(1, "id", Types.LongType.get()))
+ .fields());
+ Table table = catalog.createTable(FOO_TBL1, schema);
+ Assertions.assertThat(table.location()).contains(tempDir.toAbsolutePath().toString());
+ catalog.dropTable(FOO_TBL1);
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+
+ @Test
+ public void testBuildNessieCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("ref", "main");
+ properties.put("uri", "http://localhost:19120/api/v1");
+ properties.put("enable-api-compatibility-check", "false");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.NESSIE, "catalogName", null, null);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(NessieCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+
+ @Test
+ public void testBuildHiveCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("type", "hive");
+ properties.put("uri", "thrift://localhost:9083");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.HIVE, "catalogName", null, null);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(HiveCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java
new file mode 100644
index 00000000..fc66d07e
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CatalogMigratorParamsTest extends AbstractTest {
+
+ @Test
+ public void testInvalidArgs() {
+ sourceCatalog =
+ CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(),
+ "source",
+ hadoopCatalogProperties(true),
+ new Configuration());
+ targetCatalog =
+ CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(),
+ "target",
+ hadoopCatalogProperties(true),
+ new Configuration());
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build()
+ .registerTable(null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Identifier is null");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(null) // target-catalog is null
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("targetCatalog");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(null) // source-catalog is null
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("sourceCatalog");
+
+ // test source catalog as hadoop with `deleteEntriesFromSourceCatalog` as true.
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "Source catalog is a Hadoop catalog and it doesn't support deleting the table entries just from the catalog. "
+ + "Please configure `deleteEntriesFromSourceCatalog` as `false`");
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java
new file mode 100644
index 00000000..81ac4d5d
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class CustomCatalogMigratorTest extends AbstractTest {
+
+ protected static @TempDir Path warehouse1;
+ protected static @TempDir Path warehouse2;
+
+ @BeforeAll
+ protected static void setup() {
+ sourceCatalog = createCustomCatalog(warehouse1.toAbsolutePath().toString(), "sourceCatalog");
+ targetCatalog = createCustomCatalog(warehouse2.toAbsolutePath().toString(), "targetCatalog");
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ dropTables();
+ }
+
+ @Test
+ public void testRegister() {
+ CatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+ // should fail to register as catalog doesn't support register table operations.
+ catalogMigrator.getMatchingTableIdentifiers(null).forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+ }
+
+ private static Catalog createCustomCatalog(String warehousePath, String name) {
+
+ class TestCatalog extends HadoopCatalog {
+ @Override
+ public Table registerTable(TableIdentifier identifier, String metadataFileLocation) {
+ throw new UnsupportedOperationException("This catalog doesn't support register table");
+ }
+ }
+
+ Map properties = new HashMap<>();
+ properties.put("warehouse", warehousePath);
+ TestCatalog testCatalog = new TestCatalog();
+ testCatalog.setConf(new Configuration());
+ testCatalog.initialize(name, properties);
+ return testCatalog;
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
new file mode 100644
index 00000000..32429624
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class HadoopCatalogMigratorTest extends AbstractTestCatalogMigrator {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespaces() {
+ List namespaceList = Arrays.asList(NS1, NS2, NS3, NS1_NS2, NS1_NS3, NS1_NS2_NS3);
+
+ List identifiers =
+ Arrays.asList(TBL, NS1_TBL, NS2_TBL, NS3_TBL, NS1_NS2_TBL, NS1_NS3_TBL, NS1_NS2_NS3_TBL);
+
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::createNamespace);
+ identifiers.forEach(identifier -> sourceCatalog.createTable(identifier, schema));
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ // HadoopCatalog supports implicit namespaces.
+ // Hence, No concept of empty namespace too. So, cannot list the tables from empty
+ // namespaces.
+ // Can only load tables in empty namespace using identifiers.
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(matchingTableIdentifiers).doesNotContain(identifiers.get(0));
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // manually register the table from empty namespace
+ result = catalogMigratorWithDefaultArgs(false).registerTable(TBL).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Collections.reverse(namespaceList);
+ identifiers.forEach(sourceCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ identifiers.forEach(targetCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testCreateAndListNamespaces() {
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(4)); // try creating "a.b.c.d.e"
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(5)); // try creating "a.c"
+ // should create all the levels of missing namespaces on target catalog
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces())
+ .contains(namespaceList.get(0))
+ .doesNotContainAnyElementsOf(namespaceList.subList(1, 6));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(0)))
+ .containsExactlyInAnyOrder(namespaceList.get(1), namespaceList.get(5));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(1)))
+ .containsExactly(namespaceList.get(2));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(2)))
+ .containsExactly(namespaceList.get(3));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(3)))
+ .containsExactly(namespaceList.get(4));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(4)))
+ .isEmpty();
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(5)))
+ .isEmpty();
+
+ namespaceList.forEach(
+ namespace -> ((SupportsNamespaces) sourceCatalog).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java
new file mode 100644
index 00000000..87708c23
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHadoopToHiveCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespace() {
+ TableIdentifier tableIdentifier = TableIdentifier.of(NS_A_B_C, "tbl5");
+ // create namespace "a.b.c" only in source catalog
+ ((SupportsNamespaces) sourceCatalog).createNamespace(NS_A_B_C);
+ sourceCatalog.createTable(tableIdentifier, schema);
+
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(false).registerTable(tableIdentifier).result();
+
+ // hive catalog doesn't support multipart namespace. Hence, table should fail to register.
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers())
+ .containsExactly(tableIdentifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ sourceCatalog.dropTable(tableIdentifier);
+ ((SupportsNamespaces) sourceCatalog).dropNamespace(NS_A_B_C);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java
new file mode 100644
index 00000000..87e0034e
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class ITHadoopToNessieCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespaces() {
+ List namespaceList = Arrays.asList(NS1, NS2, NS3, NS1_NS2, NS1_NS3, NS1_NS2_NS3);
+ List identifiers =
+ Arrays.asList(TBL, NS1_TBL, NS2_TBL, NS3_TBL, NS1_NS2_TBL, NS1_NS3_TBL, NS1_NS2_NS3_TBL);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::createNamespace);
+ identifiers.forEach(identifier -> sourceCatalog.createTable(identifier, schema));
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ // HadoopCatalog supports implicit namespaces.
+ // Hence, No concept of empty namespace too. So, cannot list the tables from default
+ // namespaces.
+ // Can only load tables in empty namespace using identifiers.
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(matchingTableIdentifiers).doesNotContain(identifiers.get(0));
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // manually register the table from empty namespace
+ result = catalogMigratorWithDefaultArgs(false).registerTable(TBL).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Collections.reverse(namespaceList);
+ identifiers.forEach(sourceCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ identifiers.forEach(targetCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testCreateMissingNamespaces() {
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(4)); // try creating "a.b.c.d.e"
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(5)); // try creating "a.c"
+
+ // should create all the levels of missing namespaces on target catalog.
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces()).contains(NS_A);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A))
+ .contains(NS_A_B, NS_A_C);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B))
+ .contains(NS_A_B_C);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B_C))
+ .contains(NS_A_B_C_D);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B_C_D))
+ .contains(NS_A_B_C_D_E);
+
+ namespaceList.forEach(
+ namespace -> ((SupportsNamespaces) sourceCatalog).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testListingNamespacesFromNessie() {
+ Catalog nessie = targetCatalog;
+ Catalog hadoop = sourceCatalog;
+
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(nessie)
+ .targetCatalog(hadoop)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+
+ namespaceList.forEach(namespace -> ((SupportsNamespaces) nessie).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) nessie)::dropNamespace);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java
new file mode 100644
index 00000000..7ad97960
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToHadoopCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java
new file mode 100644
index 00000000..c7ea8430
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToNessieCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java
new file mode 100644
index 00000000..cc6e0201
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import java.util.Set;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITNessieToHiveCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithDefaultNamespace() {
+ sourceCatalog.createTable(TBL, schema);
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ // should also include table from empty namespace
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ Assertions.assertThat(matchingTableIdentifiers).contains(TBL);
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ // hive will not support empty namespace (namespace with level = 0). Hence, register will
+ // fail.
+ Assertions.assertThat(result.registeredTableIdentifiers()).doesNotContain(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java
new file mode 100644
index 00000000..66127213
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class UnsupportedNamespaceTest {
+
+ protected static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", tempDir.toAbsolutePath().toString());
+ }
+
+ @Test
+ public void testUnsupportedNamespace() {
+
+ class TestCatalog extends BaseMetastoreCatalog {
+ // doesn't support namespaces
+ @Override
+ protected TableOperations newTableOps(TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ public List listTables(Namespace namespace) {
+ return null;
+ }
+
+ @Override
+ public boolean dropTable(TableIdentifier identifier, boolean purge) {
+ return false;
+ }
+
+ @Override
+ public void renameTable(TableIdentifier from, TableIdentifier to) {}
+ }
+
+ Catalog sourceCatalog = new TestCatalog();
+ Catalog targetCatalog = new TestCatalog();
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "target catalog TestCatalog{} doesn't implement SupportsNamespaces to create missing namespaces.");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(new HadoopCatalog())
+ .deleteEntriesFromSourceCatalog(false)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "source catalog TestCatalog{} doesn't implement SupportsNamespaces to list all namespaces.");
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/resources/logback.xml b/iceberg-catalog-migrator/api/src/test/resources/logback.xml
new file mode 100644
index 00000000..769b6180
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/resources/logback.xml
@@ -0,0 +1,34 @@
+
+
+
+
+
+
+ ${catalog.migration.log.dir}/catalog_migration.log
+ true
+
+ %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n
+
+
+
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/cli/build.gradle.kts b/iceberg-catalog-migrator/cli/build.gradle.kts
new file mode 100644
index 00000000..b7534127
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/build.gradle.kts
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+ `java-library`
+ `maven-publish`
+ signing
+ alias(libs.plugins.nessie.run)
+ `build-conventions`
+}
+
+java.sourceCompatibility = JavaVersion.VERSION_21
+
+applyShadowJar()
+
+dependencies {
+ implementation(project(":iceberg-catalog-migrator-api"))
+ implementation(libs.guava)
+ implementation(libs.slf4j)
+ runtimeOnly(libs.logback.classic)
+ implementation(libs.picocli)
+ implementation(libs.iceberg.spark.runtime)
+ implementation(libs.hadoop.aws) { exclude("com.amazonaws", "aws-java-sdk-bundle") }
+ // AWS dependencies based on https://iceberg.apache.org/docs/latest/aws/#enabling-aws-integration
+ runtimeOnly(libs.aws.sdk.apache.client)
+ runtimeOnly(libs.aws.sdk.auth)
+ runtimeOnly(libs.aws.sdk.glue)
+ runtimeOnly(libs.aws.sdk.s3)
+ runtimeOnly(libs.aws.sdk.dynamo)
+ runtimeOnly(libs.aws.sdk.kms)
+ runtimeOnly(libs.aws.sdk.lakeformation)
+ runtimeOnly(libs.aws.sdk.sts)
+ runtimeOnly(libs.aws.sdk.url.connection.client)
+
+ // needed for Hive catalog
+ runtimeOnly("org.apache.hive:hive-metastore:${libs.versions.hive.get()}") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hbase")
+ exclude("org.apache.logging.log4j")
+ exclude("co.cask.tephra")
+ exclude("com.google.code.findbugs", "jsr305")
+ exclude("org.eclipse.jetty.aggregate", "jetty-all")
+ exclude("org.eclipse.jetty.orbit", "javax.servlet")
+ exclude("org.apache.parquet", "parquet-hadoop-bundle")
+ exclude("com.tdunning", "json")
+ exclude("javax.transaction", "transaction-api")
+ exclude("com.zaxxer", "HikariCP")
+ }
+ runtimeOnly("org.apache.hive:hive-exec:${libs.versions.hive.get()}:core") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hive", "hive-llap-tez")
+ exclude("org.apache.logging.log4j")
+ exclude("com.google.protobuf", "protobuf-java")
+ exclude("org.apache.calcite")
+ exclude("org.apache.calcite.avatica")
+ exclude("com.google.code.findbugs", "jsr305")
+ }
+ runtimeOnly("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
+
+ testImplementation(platform(libs.junit.bom))
+ testImplementation("org.junit.jupiter:junit-jupiter-params")
+ testImplementation("org.junit.jupiter:junit-jupiter-api")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
+ testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+ testImplementation(libs.assertj)
+ testImplementation(libs.logcaptor)
+
+ testImplementation(project(":iceberg-catalog-migrator-api-test"))
+
+ // for integration tests
+ testImplementation(
+ "org.apache.iceberg:iceberg-hive-metastore:${libs.versions.iceberg.get()}:tests"
+ )
+ // this junit4 dependency is needed for above Iceberg's TestHiveMetastore
+ testRuntimeOnly("junit:junit:4.13.2")
+
+ testImplementation("org.apache.hive:hive-metastore:${libs.versions.hive.get()}") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hbase")
+ exclude("org.apache.logging.log4j")
+ exclude("co.cask.tephra")
+ exclude("com.google.code.findbugs", "jsr305")
+ exclude("org.eclipse.jetty.aggregate", "jetty-all")
+ exclude("org.eclipse.jetty.orbit", "javax.servlet")
+ exclude("org.apache.parquet", "parquet-hadoop-bundle")
+ exclude("com.tdunning", "json")
+ exclude("javax.transaction", "transaction-api")
+ exclude("com.zaxxer", "HikariCP")
+ }
+ testImplementation("org.apache.hive:hive-exec:${libs.versions.hive.get()}:core") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hive", "hive-llap-tez")
+ exclude("org.apache.logging.log4j")
+ exclude("com.google.protobuf", "protobuf-java")
+ exclude("org.apache.calcite")
+ exclude("org.apache.calcite.avatica")
+ exclude("com.google.code.findbugs", "jsr305")
+ }
+ testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
+
+ nessieQuarkusServer(
+ "org.projectnessie.nessie:nessie-quarkus:${libs.versions.nessie.get()}:runner"
+ )
+}
+
+nessieQuarkusApp { includeTask(tasks.named("intTest")) }
+
+tasks.named("test") { systemProperty("expectedCLIVersion", project.version) }
+
+val processResources =
+ tasks.named("processResources") {
+ inputs.property("projectVersion", project.version)
+ filter(
+ org.apache.tools.ant.filters.ReplaceTokens::class,
+ mapOf("tokens" to mapOf("projectVersion" to project.version)),
+ )
+ }
+
+val mainClassName = "org.apache.polaris.iceberg.catalog.migrator.cli.CatalogMigrationCLI"
+
+val shadowJar = tasks.named("shadowJar") { isZip64 = true }
+
+shadowJar { manifest { attributes["Main-Class"] = mainClassName } }
+
+tasks.withType().configureEach { systemProperty("java.security.manager", "allow") }
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java
new file mode 100644
index 00000000..cef41dea
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.base.Preconditions;
+import java.io.Console;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationResult;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+public abstract class BaseRegisterCommand implements Callable {
+
+ @CommandLine.ArgGroup(
+ exclusive = false,
+ multiplicity = "1",
+ heading = "Source catalog options: %n")
+ protected SourceCatalogOptions sourceCatalogOptions;
+
+ @CommandLine.ArgGroup(
+ exclusive = false,
+ multiplicity = "1",
+ heading = "Target catalog options: %n")
+ private TargetCatalogOptions targetCatalogOptions;
+
+ @CommandLine.ArgGroup(heading = "Identifier options: %n")
+ private IdentifierOptions identifierOptions;
+
+ @CommandLine.Option(
+ names = {"--output-dir"},
+ defaultValue = "",
+ description = {
+ "Optional local output directory path to write CLI output files like `failed_identifiers.txt`, "
+ + "`failed_to_delete_at_source.txt`, `dry_run_identifiers.txt`. "
+ + "If not specified, uses the present working directory.",
+ "Example: --output-dir /tmp/output/",
+ " --output-dir $PWD/output_folder"
+ })
+ private Path outputDirPath;
+
+ @CommandLine.Option(
+ names = {"--dry-run"},
+ description =
+ "Optional configuration to simulate the registration without actually registering. Can learn about a list "
+ + "of tables that will be registered by running this.")
+ private boolean isDryRun;
+
+ @CommandLine.Option(
+ names = {"--disable-safety-prompts"},
+ description = "Optional configuration to disable safety prompts which needs console input.")
+ private boolean disablePrompts;
+
+ @CommandLine.Option(
+ names = {"--stacktrace"},
+ description =
+ "Optional configuration to enable capturing stacktrace in logs in case of failures.")
+ private boolean enableStackTrace;
+
+ private static final int BATCH_SIZE = 100;
+ public static final String FAILED_IDENTIFIERS_FILE = "failed_identifiers.txt";
+ public static final String FAILED_TO_DELETE_AT_SOURCE_FILE = "failed_to_delete_at_source.txt";
+ public static final String DRY_RUN_FILE = "dry_run_identifiers.txt";
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ public BaseRegisterCommand() {}
+
+ protected abstract CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace);
+
+ protected abstract boolean canProceed(Catalog sourceCatalog);
+
+ protected abstract String operation();
+
+ protected abstract String operated();
+
+ protected abstract String operate();
+
+ @Override
+ public Integer call() {
+ Set identifiers = Collections.emptySet();
+ String identifierRegEx = identifierOptions != null ? identifierOptions.identifiersRegEx : null;
+
+ if (identifierOptions != null) {
+ identifiers = identifierOptions.processIdentifiersInput();
+ }
+ checkAndWarnAboutIdentifiers(identifiers, identifierRegEx);
+
+ validateOutputDir();
+
+ Catalog sourceCatalog = null;
+ Catalog targetCatalog = null;
+
+ try {
+ sourceCatalog = sourceCatalogOptions.build();
+ consoleLog.info("Configured source catalog: {}", sourceCatalog.name());
+
+ targetCatalog = targetCatalogOptions.build();
+ consoleLog.info("Configured target catalog: {}", targetCatalog.name());
+
+ if (!isDryRun && !disablePrompts && !canProceed(sourceCatalog)) {
+ return 1;
+ }
+
+ CatalogMigrator catalogMigrator =
+ catalogMigrator(sourceCatalog, targetCatalog, enableStackTrace);
+
+ if (identifiers.isEmpty()) {
+ consoleLog.info("Identifying tables for {} ...", operation());
+ identifiers = catalogMigrator.getMatchingTableIdentifiers(identifierRegEx);
+ if (identifiers.isEmpty()) {
+ consoleLog.warn(
+ "No tables were identified for {}. Please check `catalog_migration.log` file for more info.",
+ operation());
+ return 1;
+ }
+ }
+
+ if (isDryRun) {
+ consoleLog.info("Dry run is completed.");
+ handleDryRunResult(identifiers);
+ return 0;
+ }
+
+ consoleLog.info("Identified {} tables for {}.", identifiers.size(), operation());
+
+ consoleLog.info("Started {} ...", operation());
+
+ CatalogMigrationResult result;
+ try {
+ int processedIdentifiersCount = 0;
+ for (TableIdentifier identifier : identifiers) {
+ catalogMigrator.registerTable(identifier);
+ processedIdentifiersCount++;
+ if (processedIdentifiersCount % BATCH_SIZE == 0
+ || processedIdentifiersCount == identifiers.size()) {
+ consoleLog.info(
+ "Attempted {} for {} tables out of {} tables.",
+ operation(),
+ processedIdentifiersCount,
+ identifiers.size());
+ }
+ }
+ } finally {
+ consoleLog.info("Finished {} ...", operation());
+ result = catalogMigrator.result();
+ handleResults(result);
+ }
+
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()
+ || !result.failedToDeleteTableIdentifiers().isEmpty()
+ || result.registeredTableIdentifiers().isEmpty()) {
+ return 1;
+ }
+
+ return 0;
+ } finally {
+ close(sourceCatalog);
+ close(targetCatalog);
+ }
+ }
+
+ private void close(Catalog catalog) {
+ if (catalog instanceof AutoCloseable) {
+ try {
+ ((AutoCloseable) catalog).close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void checkAndWarnAboutIdentifiers(
+ Set identifiers, String identifierRegEx) {
+ if (identifiers.isEmpty()) {
+ if (identifierRegEx != null) {
+ consoleLog.warn(
+ "User has not specified the table identifiers."
+ + " Will be selecting all the tables from all the namespaces from the source catalog "
+ + "which matches the regex pattern:{}",
+ identifierRegEx);
+ } else {
+ consoleLog.warn(
+ "User has not specified the table identifiers."
+ + " Will be selecting all the tables from all the namespaces from the source catalog.");
+ }
+ }
+ }
+
+ private void validateOutputDir() {
+ if (!Files.exists(outputDirPath)) {
+ try {
+ Files.createDirectories(outputDirPath);
+ } catch (IOException ex) {
+ throw new UncheckedIOException(
+ "Failed to create the output directory from the path specified in `--output-dir`", ex);
+ }
+ }
+ Preconditions.checkArgument(
+ Files.isWritable(outputDirPath), "Path specified in `--output-dir` is not writable");
+ }
+
+ private void handleResults(CatalogMigrationResult result) {
+ try {
+ writeToFile(
+ outputDirPath.resolve(FAILED_IDENTIFIERS_FILE),
+ result.failedToRegisterTableIdentifiers());
+ writeToFile(
+ outputDirPath.resolve(FAILED_TO_DELETE_AT_SOURCE_FILE),
+ result.failedToDeleteTableIdentifiers());
+ } finally {
+ printSummary(result);
+ printDetails(result);
+ }
+ }
+
+ private void handleDryRunResult(Set identifiers) {
+ try {
+ writeToFile(outputDirPath.resolve(DRY_RUN_FILE), identifiers);
+ } finally {
+ printDryRunResult(identifiers);
+ }
+ }
+
+ private void printSummary(CatalogMigrationResult result) {
+ consoleLog.info("Summary: ");
+ if (!result.registeredTableIdentifiers().isEmpty()) {
+ consoleLog.info(
+ "Successfully {} {} tables from {} catalog to {} catalog.",
+ operated(),
+ result.registeredTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ targetCatalogOptions.type.name());
+ }
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to {} {} tables from {} catalog to {} catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `{}`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operate(),
+ result.failedToRegisterTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ targetCatalogOptions.type.name(),
+ FAILED_IDENTIFIERS_FILE);
+ }
+ if (!result.failedToDeleteTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to delete {} tables from {} catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "{}Failed to delete identifiers are written into `{}`.",
+ result.failedToDeleteTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ System.lineSeparator(),
+ FAILED_TO_DELETE_AT_SOURCE_FILE);
+ }
+ }
+
+ private void printDetails(CatalogMigrationResult result) {
+ consoleLog.info("Details: ");
+ if (!result.registeredTableIdentifiers().isEmpty()) {
+ consoleLog.info(
+ "Successfully {} these tables:{}{}",
+ operated(),
+ System.lineSeparator(),
+ result.registeredTableIdentifiers());
+ }
+
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to {} these tables:{}{}",
+ operate(),
+ System.lineSeparator(),
+ result.failedToRegisterTableIdentifiers());
+ }
+
+ if (!result.failedToDeleteTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to delete these tables from source catalog:{}{}",
+ System.lineSeparator(),
+ result.failedToDeleteTableIdentifiers());
+ }
+ }
+
+ private void printDryRunResult(Set result) {
+ consoleLog.info("Summary: ");
+ consoleLog.info(
+ "Identified {} tables for {} by dry-run. These identifiers are also written into {}. "
+ + "This file can be used with `--identifiers-from-file` option for an actual run.",
+ result.size(),
+ operation(),
+ DRY_RUN_FILE);
+ consoleLog.info(
+ "Details: {}Identified these tables for {} by dry-run:{}{}",
+ System.lineSeparator(),
+ operation(),
+ System.lineSeparator(),
+ result);
+ }
+
+ private static void writeToFile(Path filePath, Collection identifiers) {
+ List identifiersString =
+ identifiers.stream().map(TableIdentifier::toString).collect(Collectors.toList());
+ try {
+ Files.write(filePath, identifiersString);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write the file:" + filePath, e);
+ }
+ }
+
+ protected boolean proceed() {
+ Console console = System.console();
+ while (true) {
+ consoleLog.info(
+ "Are you certain that you wish to proceed, after reading the above warnings? (yes/no):");
+ String input = console.readLine();
+
+ if (input.equalsIgnoreCase("yes")) {
+ consoleLog.info("Continuing...");
+ return true;
+ } else if (input.equalsIgnoreCase("no")) {
+ consoleLog.info("Aborting...");
+ return false;
+ } else {
+ consoleLog.info("Invalid input. Please enter 'yes' or 'no'.");
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java
new file mode 100644
index 00000000..3fecc2d4
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.io.InputStream;
+import java.util.Properties;
+import picocli.CommandLine.IVersionProvider;
+
+public class CLIVersionProvider implements IVersionProvider {
+ @Override
+ public String[] getVersion() throws Exception {
+ try (InputStream input =
+ CLIVersionProvider.class
+ .getResource("version.properties")
+ .openConnection()
+ .getInputStream()) {
+ Properties props = new Properties();
+ props.load(input);
+ return new String[] {props.getProperty("cli.version")};
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java
new file mode 100644
index 00000000..6d4b0820
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "iceberg-catalog-migrator",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ subcommands = {MigrateCommand.class, RegisterCommand.class})
+public class CatalogMigrationCLI {
+
+ public CatalogMigrationCLI() {}
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ public static void main(String... args) {
+ CommandLine commandLine =
+ new CommandLine(new CatalogMigrationCLI())
+ .setExecutionExceptionHandler(
+ (ex, cmd, parseResult) -> {
+ if (enableStacktrace(args)) {
+ cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex));
+ } else {
+ consoleLog.error(
+ "Error during CLI execution: {}. Please check `catalog_migration.log` file for more info.",
+ ex.getMessage());
+ }
+ return 1;
+ });
+ commandLine.setUsageHelpWidth(150);
+ int exitCode = commandLine.execute(args);
+ System.exit(exitCode);
+ }
+
+ private static boolean enableStacktrace(String... args) {
+ for (String arg : args) {
+ if (arg.equalsIgnoreCase("--stacktrace")) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java
new file mode 100644
index 00000000..7047935b
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+public class IdentifierOptions {
+
+ @CommandLine.Option(
+ names = {"--identifiers"},
+ split = ",",
+ description = {
+ "Optional selective set of identifiers to register. If not specified, all the tables will be registered. "
+ + "Use this when there are few identifiers that need to be registered. For a large number of identifiers, "
+ + "use the `--identifiers-from-file` or `--identifiers-regex` option.",
+ "Example: --identifiers foo.t1,bar.t2"
+ })
+ protected Set identifiers = new HashSet<>();
+
+ @CommandLine.Option(
+ names = {"--identifiers-from-file"},
+ description = {
+ "Optional text file path that contains a set of table identifiers (one per line) to register. Should not be "
+ + "used with `--identifiers` or `--identifiers-regex` option.",
+ "Example: --identifiers-from-file /tmp/files/ids.txt"
+ })
+ protected String identifiersFromFile;
+
+ @CommandLine.Option(
+ names = {"--identifiers-regex"},
+ description = {
+ "Optional regular expression pattern used to register only the tables whose identifiers match this pattern. "
+ + "Should not be used with `--identifiers` or '--identifiers-from-file' option.",
+ "Example: --identifiers-regex ^foo\\..*"
+ })
+ protected String identifiersRegEx;
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ protected Set processIdentifiersInput() {
+
+ if (!identifiers.isEmpty()) {
+ return identifiers.stream()
+ .map(TableIdentifier::parse)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ } else if (identifiersFromFile != null) {
+ Preconditions.checkArgument(
+ Files.exists(Paths.get(identifiersFromFile)),
+ "File specified in `--identifiers-from-file` option does not exist");
+ try {
+ consoleLog.info("Collecting identifiers from the file {} ...", identifiersFromFile);
+ return Files.readAllLines(Paths.get(identifiersFromFile)).stream()
+ .map(String::trim)
+ .filter(string -> !string.isEmpty())
+ .map(TableIdentifier::parse)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ String.format("Failed to read the file: %s", identifiersFromFile), e);
+ }
+ } else if (identifiersRegEx != null) {
+ Preconditions.checkArgument(
+ !identifiersRegEx.trim().isEmpty(), "--identifiers-regex should not be empty");
+ // check whether pattern is compilable
+ try {
+ Pattern.compile(identifiersRegEx);
+ } catch (PatternSyntaxException ex) {
+ throw new IllegalArgumentException("--identifiers-regex pattern is not compilable", ex);
+ }
+ }
+ return Sets.newHashSet();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java
new file mode 100644
index 00000000..5616e07c
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.ImmutableCatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "migrate",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ // As both source and target catalog has similar configurations,
+ // documentation is easy to read if the target and source property is one after another instead
+ // of sorted order.
+ sortOptions = false,
+ description =
+ "Bulk migrate the iceberg tables from source catalog to target catalog without data copy."
+ + " Table entries from source catalog will be deleted after the successful migration to the target "
+ + "catalog.")
+public class MigrateCommand extends BaseRegisterCommand {
+
+ private static final String newLine = System.lineSeparator();
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @Override
+ protected CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace) {
+
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .enableStacktrace(enableStackTrace)
+ .build();
+ }
+
+ @Override
+ public Integer call() {
+ if (sourceCatalogOptions.type == CatalogMigrationUtil.CatalogType.HADOOP) {
+ consoleLog.error(
+ "Source catalog is a Hadoop catalog and it doesn't support deleting the table entries just from the catalog. "
+ + "Please use 'register' command instead.");
+ return 1;
+ }
+ return super.call();
+ }
+
+ @Override
+ protected boolean canProceed(Catalog sourceCatalog) {
+ consoleLog.warn(
+ "{}"
+ + "\ta) Executing catalog migration when the source catalog has some in-progress commits "
+ + "{}\tcan lead to a data loss as the in-progress commits will not be considered for migration. "
+ + "{}\tSo, while using this tool please make sure there are no in-progress commits for the source "
+ + "catalog.{}"
+ + "{}"
+ + "\tb) After the migration, successfully migrated tables will be deleted from the source catalog "
+ + "{}\tand can only be accessed from the target catalog.",
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine);
+ return proceed();
+ }
+
+ @Override
+ protected String operation() {
+ return "migration";
+ }
+
+ @Override
+ protected String operated() {
+ return "migrated";
+ }
+
+ @Override
+ protected String operate() {
+ return "migrate";
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java
new file mode 100644
index 00000000..c482217a
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.ImmutableCatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "register",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ // As both source and target catalog has similar configurations,
+ // documentation is easy to read if the target and source property is one after another instead
+ // of sorted order.
+ sortOptions = false,
+ description =
+ "Bulk register the iceberg tables from source catalog to target catalog without data copy.")
+public class RegisterCommand extends BaseRegisterCommand {
+
+ private static final String newLine = System.lineSeparator();
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @Override
+ protected CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace) {
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .enableStacktrace(enableStackTrace)
+ .build();
+ }
+
+ @Override
+ protected boolean canProceed(Catalog sourceCatalog) {
+ consoleLog.warn(
+ "{}"
+ + "\ta) Executing catalog migration when the source catalog has some in-progress commits "
+ + "{}\tcan lead to a data loss as the in-progress commits will not be considered for migration. "
+ + "{}\tSo, while using this tool please make sure there are no in-progress commits for the source "
+ + "catalog.{}"
+ + "{}"
+ + "\tb) After the registration, successfully registered tables will be present in both source and target "
+ + "catalog. "
+ + "{}\tHaving the same metadata.json registered in more than one catalog can lead to missing updates, "
+ + "loss of data, and table corruption. "
+ + "{}\tUse `migrate` command to automatically delete the table from source catalog after "
+ + "migration.",
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine);
+ return proceed();
+ }
+
+ @Override
+ protected String operation() {
+ return "registration";
+ }
+
+ @Override
+ protected String operated() {
+ return "registered";
+ }
+
+ @Override
+ protected String operate() {
+ return "register";
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java
new file mode 100644
index 00000000..31e2c248
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import picocli.CommandLine;
+
+public class SourceCatalogOptions {
+
+ @CommandLine.Option(
+ names = "--source-catalog-type",
+ required = true,
+ description = {
+ "Source catalog type. Can be one of these [CUSTOM, DYNAMODB, ECS, GLUE, HADOOP, HIVE, JDBC, "
+ + "NESSIE, REST].",
+ "Example: --source-catalog-type GLUE",
+ " --source-catalog-type NESSIE"
+ })
+ protected CatalogMigrationUtil.CatalogType type;
+
+ @CommandLine.Option(
+ names = "--source-catalog-properties",
+ required = true,
+ split = ",",
+ description = {
+ "Iceberg catalog properties for source catalog (like uri, warehouse, etc).",
+ "Example: --source-catalog-properties uri=http://localhost:19120/api/v1,ref=main,warehouse=/tmp/warehouseNessie"
+ })
+ private Map properties;
+
+ @CommandLine.Option(
+ names = "--source-catalog-hadoop-conf",
+ split = ",",
+ description = {
+ "Optional source catalog Hadoop configurations required by the Iceberg catalog.",
+ "Example: --source-catalog-hadoop-conf key1=value1,key2=value2"
+ })
+ private final Map hadoopConf = new HashMap<>();
+
+ @CommandLine.Option(
+ names = {"--source-custom-catalog-impl"},
+ description = {
+ "Optional fully qualified class name of the custom catalog implementation of the source catalog. Required "
+ + "when the catalog type is CUSTOM.",
+ "Example: --source-custom-catalog-impl org.apache.iceberg.AwesomeCatalog"
+ })
+ private String customCatalogImpl;
+
+ Catalog build() {
+ return CatalogMigrationUtil.buildCatalog(
+ properties, type, "SOURCE_CATALOG_" + type.name(), customCatalogImpl, hadoopConf);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java
new file mode 100644
index 00000000..1ea9bc9d
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import picocli.CommandLine;
+
+public class TargetCatalogOptions {
+
+ @CommandLine.Option(
+ names = "--target-catalog-type",
+ required = true,
+ description = {
+ "Target catalog type. Can be one of these [CUSTOM, DYNAMODB, ECS, GLUE, HADOOP, HIVE, JDBC, "
+ + "NESSIE, REST].",
+ "Example: --target-catalog-type GLUE",
+ " --target-catalog-type NESSIE"
+ })
+ protected CatalogMigrationUtil.CatalogType type;
+
+ @CommandLine.Option(
+ names = "--target-catalog-properties",
+ required = true,
+ split = ",",
+ description = {
+ "Iceberg catalog properties for target catalog (like uri, warehouse, etc).",
+ "Example: --target-catalog-properties uri=http://localhost:19120/api/v1,ref=main,warehouse=/tmp/warehouseNessie"
+ })
+ private Map properties;
+
+ @CommandLine.Option(
+ names = "--target-catalog-hadoop-conf",
+ split = ",",
+ description = {
+ "Optional target catalog Hadoop configurations required by the Iceberg catalog.",
+ "Example: --target-catalog-hadoop-conf key1=value1,key2=value2"
+ })
+ private final Map hadoopConf = new HashMap<>();
+
+ @CommandLine.Option(
+ names = {"--target-custom-catalog-impl"},
+ description = {
+ "Optional fully qualified class name of the custom catalog implementation of the target catalog. Required "
+ + "when the catalog type is CUSTOM.",
+ "Example: --target-custom-catalog-impl org.apache.iceberg.AwesomeCatalog"
+ })
+ private String customCatalogImpl;
+
+ Catalog build() {
+ return CatalogMigrationUtil.buildCatalog(
+ properties, type, "TARGET_CATALOG_" + type.name(), customCatalogImpl, hadoopConf);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/resources/logback.xml b/iceberg-catalog-migrator/cli/src/main/resources/logback.xml
new file mode 100644
index 00000000..b257baf3
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/resources/logback.xml
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+ ${catalog.migration.log.dir}/catalog_migration.log
+ true
+
+ %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n
+
+
+
+
+ true
+
+ %highlight(%-5level) - %msg%n
+
+
+
+
+
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties b/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties
new file mode 100644
index 00000000..45434cbe
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+cli.version=@projectVersion@
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
new file mode 100644
index 00000000..57f85ac7
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.apache.polaris.iceberg.catalog.migrator.cli.BaseRegisterCommand.DRY_RUN_FILE;
+import static org.apache.polaris.iceberg.catalog.migrator.cli.BaseRegisterCommand.FAILED_IDENTIFIERS_FILE;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import nl.altindag.log.LogCaptor;
+import nl.altindag.log.model.LogEvent;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public abstract class AbstractCLIMigrationTest extends AbstractTest {
+
+ protected static @TempDir Path outputDir;
+
+ protected static String sourceCatalogProperties;
+ protected static String targetCatalogProperties;
+
+ protected static String sourceCatalogType;
+ protected static String targetCatalogType;
+
+ protected static void initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(true, catalogType, additionalProp);
+ createNamespacesForSourceCatalog();
+ }
+
+ protected static void initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(false, catalogType, additionalProp);
+ createNamespacesForTargetCatalog();
+ }
+
+ private static void initializeCatalog(
+ boolean isSourceCatalog,
+ CatalogMigrationUtil.CatalogType catalogType,
+ Map additionalProp) {
+ Map properties;
+ switch (catalogType) {
+ case HADOOP:
+ properties = hadoopCatalogProperties(isSourceCatalog);
+ break;
+ case NESSIE:
+ properties = nessieCatalogProperties(isSourceCatalog);
+ break;
+ case HIVE:
+ properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported for catalog type: %s", catalogType));
+ }
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties,
+ catalogType,
+ isSourceCatalog ? "sourceCatalog" : "targetCatalog" + "_" + catalogType,
+ null,
+ null);
+ String propertiesStr = Joiner.on(",").withKeyValueSeparator("=").join(properties);
+ if (isSourceCatalog) {
+ sourceCatalog = catalog;
+ sourceCatalogProperties = propertiesStr;
+ sourceCatalogType = catalogType.name();
+ } else {
+ targetCatalog = catalog;
+ targetCatalogProperties = propertiesStr;
+ targetCatalogType = catalogType.name();
+ }
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ // create table will call refresh internally.
+ sourceCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
+ targetCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
+
+ dropTables();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegister(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(
+ "User has not specified the table identifiers. "
+ + "Will be selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 4 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 4 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(FOO_TBL1).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ sourceCatalog.tableExists(FOO_TBL1);
+
+ if (deleteSourceTables) {
+ // table should be deleted after migration from source catalog
+ Assertions.assertThat(sourceCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(sourceCatalog.listTables(BAR)).isEmpty();
+ } else {
+ // tables should be present in source catalog.
+ Assertions.assertThat(sourceCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(sourceCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterSelectedTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // using `--identifiers` option
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "bar.tbl3"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "User has not specified the table identifiers. "
+ + "Selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n[bar.tbl3]", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+
+ Path identifierFile = outputDir.resolve("ids.txt");
+
+ // using `--identifiers-from-file` option
+ Files.write(identifierFile, Collections.singletonList("bar.tbl4"));
+ argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList("--identifiers-from-file", identifierFile.toAbsolutePath().toString()));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "User has not specified the table identifiers. "
+ + "Selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL4, BAR_TBL3);
+ Files.delete(identifierFile);
+
+ // using `--identifiers-regex` option which matches all the tables starts with "foo."
+ argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers-regex", "^foo\\..*"));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(
+ "User has not specified the table identifiers. Will be selecting all the tables from all the namespaces "
+ + "from the source catalog which matches the regex pattern:^foo\\..*")
+ .contains(String.format("Identified 2 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 2 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterError(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operate = deleteSourceTables ? "migrate" : "register";
+
+ // use invalid namespace which leads to NoSuchTableException
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "dummy.tbl3"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nFailed to %s 1 tables from %s catalog to %s catalog."
+ + " Please check the `catalog_migration.log`",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[dummy.tbl3]", operate));
+
+ // try to register same table twice which leads to AlreadyExistsException
+ argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "foo.tbl2"));
+ runCLI(deleteSourceTables, argsList);
+ run = RunCLI.run(argsList.toArray(new String[0]));
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nFailed to %s 1 tables from %s catalog to %s catalog."
+ + " Please check the `catalog_migration.log`",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[foo.tbl2]", operate));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithFewFailures(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+ String operate = deleteSourceTables ? "migrate" : "register";
+
+ // register only foo.tbl2
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "foo.tbl2"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n[foo.tbl2]", operated));
+
+ if (deleteSourceTables) {
+ // create a table with the same name in source catalog which got deleted.
+ sourceCatalog.createTable(FOO_TBL2, schema);
+ }
+
+ // register all the tables from source catalog again. So that registering `foo.tbl2` will fail.
+ run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 4 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Successfully %s 3 tables from %s catalog to %s catalog.%n"
+ + "Failed to %s 1 tables from %s catalog to %s catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `failed_identifiers.txt`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operated,
+ sourceCatalogType,
+ targetCatalogType,
+ operate,
+ sourceCatalogType,
+ targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated))
+ .contains(String.format("Failed to %s these tables:%n[foo.tbl2]", operate));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ Path failedIdentifiersFile = outputDir.resolve(FAILED_IDENTIFIERS_FILE);
+
+ // retry the failed tables using `--identifiers-from-file`
+ argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList(
+ "--identifiers-from-file", failedIdentifiersFile.toAbsolutePath().toString()));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getOut())
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Failed to %s 1 tables from %s catalog to %s catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `failed_identifiers.txt`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[foo.tbl2]", operate));
+ Assertions.assertThat(failedIdentifiersFile).exists();
+ Assertions.assertThat(Files.readAllLines(failedIdentifiersFile)).containsExactly("foo.tbl2");
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterNoTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // clean up the default tables present in the source catalog.
+ dropTables();
+
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ String operation = deleteSourceTables ? "migration" : "registration";
+ Assertions.assertThat(run.getOut())
+ .contains(
+ String.format(
+ "No tables were identified for %s. Please check `catalog_migration.log` file for more info.",
+ operation));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDryRun(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ List argsList = defaultArgs();
+ argsList.add("--dry-run");
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ String operation = deleteSourceTables ? "migration" : "registration";
+ // should not prompt for dry run
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "Are you certain that you wish to proceed, after reading the above warnings? (yes/no):")
+ .contains("Dry run is completed.")
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Identified 4 tables for %s by dry-run. "
+ + "These identifiers are also written into dry_run_identifiers.txt. "
+ + "This file can be used with `--identifiers-from-file` option for an actual run.",
+ operation))
+ .contains(
+ String.format("Details: %nIdentified these tables for %s by dry-run:%n", operation));
+ Path dryRunFile = outputDir.resolve(DRY_RUN_FILE);
+ Assertions.assertThat(dryRunFile).exists();
+ Assertions.assertThat(Files.readAllLines(dryRunFile))
+ .containsExactlyInAnyOrder("foo.tbl1", "foo.tbl2", "bar.tbl3", "bar.tbl4");
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
+ public void testStacktrace(boolean deleteSourceTables, boolean enableStacktrace)
+ throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+ try (LogCaptor logCaptor = LogCaptor.forClass(CatalogMigrator.class)) {
+ List argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList("--identifiers", "db.dummy_table", "--stacktrace=" + enableStacktrace));
+ runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(logCaptor.getLogEvents()).hasSize(1);
+ LogEvent logEvent = logCaptor.getLogEvents().get(0);
+ if (enableStacktrace) {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo("Unable to register the table db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable())
+ .isPresent()
+ .get()
+ .isInstanceOf(NoSuchTableException.class);
+ } else {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo(
+ "Unable to register the table db.dummy_table : Table does not exist: db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable()).isEmpty();
+ }
+ }
+ }
+
+ protected static List defaultArgs() {
+ return Lists.newArrayList(
+ "--source-catalog-type",
+ sourceCatalogType,
+ "--source-catalog-properties",
+ sourceCatalogProperties,
+ "--target-catalog-type",
+ targetCatalogType,
+ "--target-catalog-properties",
+ targetCatalogProperties,
+ "--output-dir",
+ outputDir.toAbsolutePath().toString(),
+ "--disable-safety-prompts");
+ }
+
+ protected static RunCLI runCLI(boolean deleteSourceTables, List argsList)
+ throws Exception {
+ if (!deleteSourceTables) {
+ argsList.add(0, "register");
+ } else {
+ argsList.add(0, "migrate");
+ }
+ return RunCLI.run(argsList.toArray(new String[0]));
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java
new file mode 100644
index 00000000..0f63fbbc
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Stream;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+public class CLIOptionsTest {
+
+ protected static @TempDir Path logDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", logDir.toAbsolutePath().toString());
+ }
+
+ private static Stream optionErrors() {
+ return Stream.of(
+ // no arguments
+ arguments(
+ Lists.newArrayList(),
+ "Error: Missing required argument(s): (--target-catalog-type= --target-catalog-properties=[,...] [--target-catalog-properties=[,...]]... [--target-catalog-hadoop-conf=[,...]]... [--target-custom-catalog-impl=])"),
+ // missing required arguments
+ arguments(Lists.newArrayList(""), "Unmatched argument at index 1: ''"),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-properties", "properties1=ab", "--target-catalog-type", "NESSIE"),
+ "Error: Missing required argument(s): --source-catalog-type="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList("--source-catalog-type", "GLUE"),
+ "Error: Missing required argument(s): --source-catalog-properties="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "properties1=ab",
+ "--target-catalog-type",
+ "NESSIE"),
+ "Error: Missing required argument(s): --target-catalog-properties="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "properties1=ab",
+ "--target-catalog-properties",
+ "properties2=cd"),
+ "Error: Missing required argument(s): --target-catalog-type="),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-from-file",
+ "file.txt",
+ "--identifiers-regex",
+ "^foo\\."),
+ "Error: --identifiers=, --identifiers-from-file=, --identifiers-regex= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error: --identifiers=, --identifiers-from-file= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers-regex",
+ "^foo\\.",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error: --identifiers-from-file=, --identifiers-regex= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-regex",
+ "^foo\\."),
+ "Error: --identifiers=, --identifiers-regex= are mutually exclusive "
+ + "(specify only one)"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("optionErrors")
+ public void testOptionErrorsForRegister(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("register", args, expectedMessage, 2);
+ }
+
+ @ParameterizedTest
+ @MethodSource("optionErrors")
+ public void testOptionErrorsForMigrate(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("migrate", args, expectedMessage, 2);
+ }
+
+ private static Stream invalidArgs() {
+ return Stream.of(
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4"),
+ "Error during CLI execution: Cannot initialize HadoopCatalog "
+ + "because warehousePath must not be null or empty"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error during CLI execution: File specified in `--identifiers-from-file` option does not exist"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ "/path/to/file"),
+ "Error during CLI execution: Failed to create the output directory from the path specified in `--output-dir`"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ readOnlyDirLocation()),
+ "Error during CLI execution: Path specified in `--output-dir` is not writable"),
+ // test with stacktrace
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ readOnlyDirLocation(),
+ "--stacktrace"),
+ "java.lang.IllegalArgumentException: Path specified in `--output-dir` is not writable"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidArgs")
+ public void testInvalidArgsForRegister(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("register", args, expectedMessage, 1);
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidArgs")
+ public void testInvalidArgsForMigrate(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("migrate", args, expectedMessage, 1);
+ }
+
+ @Test
+ public void version() throws Exception {
+ RunCLI run = RunCLI.runWithPrintWriter("--version");
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut()).startsWith(System.getProperty("expectedCLIVersion"));
+ }
+
+ private static void executeAndValidateResults(
+ String command, List args, String expectedMessage, int expectedErrorCode)
+ throws Exception {
+ args.add(0, command);
+ RunCLI run = RunCLI.run(args);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(expectedErrorCode);
+ Assertions.assertThat(run.getErr()).contains(expectedMessage);
+ }
+
+ private static String readOnlyDirLocation() {
+ Path readOnly = logDir.resolve(UUID.randomUUID().toString());
+ try {
+ Files.createDirectory(readOnly);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Assertions.assertThat(readOnly.toFile().setWritable(false)).isTrue();
+
+ return readOnly.toAbsolutePath().toString();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
new file mode 100644
index 00000000..0e6d6c42
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.BeforeAll;
+
+public class HadoopCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java
new file mode 100644
index 00000000..a7f4ee5c
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHadoopToHiveCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
new file mode 100644
index 00000000..938df448
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.IntStream;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class ITHadoopToNessieCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @Test
+ public void testRegisterLargeNumberOfTablesWithNestedNamespaces() throws Exception {
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ namespaceList.forEach(
+ namespace -> {
+ ((SupportsNamespaces) sourceCatalog).createNamespace(namespace);
+ IntStream.range(0, 40)
+ .forEach(
+ val ->
+ sourceCatalog.createTable(
+ TableIdentifier.of(namespace, "tblx" + val), schema));
+ });
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(false, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains("Identified 244 tables for registration.")
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully registered 244 tables from %s catalog to" + " %s catalog.",
+ sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully registered these tables:%n"))
+ // validate intermediate output
+ .contains("Attempted registration for 100 tables out of 244 tables.")
+ .contains("Attempted registration for 200 tables out of 244 tables.")
+ .contains("Attempted registration for 244 tables out of 244 tables.");
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(
+ namespace -> {
+ List identifiers = targetCatalog.listTables(namespace);
+
+ // validate tables count in each namespace.
+ Assertions.assertThat(identifiers).hasSize(40);
+
+ identifiers.forEach(
+ identifier -> {
+ targetCatalog.dropTable(identifier);
+ sourceCatalog.dropTable(identifier);
+ });
+ ((SupportsNamespaces) sourceCatalog).dropNamespace(namespace);
+ ((SupportsNamespaces) targetCatalog).dropNamespace(namespace);
+ });
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java
new file mode 100644
index 00000000..7a8a4197
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToHadoopCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java
new file mode 100644
index 00000000..09876329
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import java.util.stream.IntStream;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class ITHiveToNessieCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterLargeNumberOfTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ IntStream.range(0, 240)
+ .forEach(val -> sourceCatalog.createTable(TableIdentifier.of(FOO, "tblx" + val), schema));
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 244 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 244 tables from %s catalog to" + " %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated))
+ // validate intermediate output
+ .contains(String.format("Attempted %s for 100 tables out of 244 tables.", operation))
+ .contains(String.format("Attempted %s for 200 tables out of 244 tables.", operation))
+ .contains(String.format("Attempted %s for 244 tables out of 244 tables.", operation));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).hasSize(242);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java
new file mode 100644
index 00000000..d151aed4
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITNessieToHiveCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java
new file mode 100644
index 00000000..482cbfd1
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.collect.Sets;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Set;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class ProcessIdentifiersTest {
+
+ protected static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", tempDir.toAbsolutePath().toString());
+ }
+
+ @Test
+ public void testIdentifiersSet() {
+ // test empty set
+ Assertions.assertThat(new IdentifierOptions().processIdentifiersInput()).isEmpty();
+
+ // test valid elements
+ IdentifierOptions identifierOptions = new IdentifierOptions();
+ identifierOptions.identifiers = Sets.newHashSet("foo.abc", "bar.def");
+ Assertions.assertThat(identifierOptions.processIdentifiersInput())
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("foo.abc"), TableIdentifier.parse("bar.def"));
+ }
+
+ @Test
+ public void testIdentifiersFromFile() throws Exception {
+ // valid file contents
+ Path identifierFile = tempDir.resolve("file_with_ids.txt");
+ Files.write(identifierFile, Arrays.asList("db1.t1", "db2.t2", "db123.t5"));
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThat(options.processIdentifiersInput())
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("db1.t1"),
+ TableIdentifier.parse("db2.t2"),
+ TableIdentifier.parse("db123.t5"));
+
+ // empty file
+ identifierFile = tempDir.resolve("ids1.txt");
+ Files.createFile(identifierFile);
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThat(options.processIdentifiersInput()).isEmpty();
+
+ // file with some blanks contents
+ identifierFile = tempDir.resolve("ids2.txt");
+ String[] lines = {"abc. def", " abc 123 ", "", "", " xyz%n123"};
+ Files.writeString(identifierFile, String.join(System.lineSeparator(), lines));
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Set identifiers = options.processIdentifiersInput();
+ Assertions.assertThat(identifiers)
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("abc. def"),
+ TableIdentifier.parse("abc 123"),
+ TableIdentifier.parse("xyz%n123"));
+
+ // with duplicate entries
+ identifierFile = tempDir.resolve("ids3.txt");
+ String[] ids = {"abc.def", "xx.yy", "abc.def", "abc.def", "abc.def ", " xx.yy"};
+ Files.writeString(identifierFile, String.join(System.lineSeparator(), ids));
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ identifiers = options.processIdentifiersInput();
+ Assertions.assertThat(identifiers)
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("abc.def"), TableIdentifier.parse("xx.yy"));
+ }
+
+ @Test
+ public void testIdentifiersFromFileInvalidInputs() throws Exception {
+ // file without permission to read
+ Path identifierFile = tempDir.resolve("non_readable_file.txt");
+ Files.createFile(identifierFile);
+ Assertions.assertThat(identifierFile.toFile().setReadable(false)).isTrue();
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(UncheckedIOException.class)
+ .hasMessageContaining("Failed to read the file: " + identifierFile);
+ Assertions.assertThat(identifierFile.toFile().setReadable(true)).isTrue();
+
+ // file doesn't exist
+ options.identifiersFromFile = "path/to/file";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("File specified in `--identifiers-from-file` option does not exist");
+ }
+
+ @Test
+ public void testIdentifiersRegEx() {
+ // test valid regex
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersRegEx = "^foo\\..*";
+ Assertions.assertThat(options.processIdentifiersInput()).isEmpty();
+
+ // test invalid regex
+ options.identifiersRegEx = "(23erf423!";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("--identifiers-regex pattern is not compilable");
+
+ options = new IdentifierOptions();
+ options.identifiersRegEx = " ";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("--identifiers-regex should not be empty");
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java
new file mode 100644
index 00000000..0e17c605
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.List;
+import nl.altindag.log.LogCaptor;
+import picocli.CommandLine;
+
+/** Helper class for tests. */
+public final class RunCLI {
+
+ private final String[] args;
+ private final int exitCode;
+ private final String out;
+ private final String err;
+
+ public RunCLI(int exitCode, String out, String err, String[] args) {
+ this.args = args;
+ this.exitCode = exitCode;
+ this.out = out;
+ this.err = err;
+ }
+
+ public static RunCLI run(List args) throws Exception {
+ return run(args.toArray(new String[0]));
+ }
+
+ public static RunCLI run(String... args) throws Exception {
+ try (LogCaptor logCaptor = LogCaptor.forName("console-log");
+ StringWriter err = new StringWriter();
+ PrintWriter errWriter = new PrintWriter(err)) {
+ int exitCode = runMain(null, errWriter, args);
+ String out = String.join(System.lineSeparator(), logCaptor.getLogs());
+ return new RunCLI(exitCode, out, err.toString(), args);
+ }
+ }
+
+ public static RunCLI runWithPrintWriter(String... args) throws Exception {
+ try (StringWriter out = new StringWriter();
+ PrintWriter outWriter = new PrintWriter(out);
+ StringWriter err = new StringWriter();
+ PrintWriter errWriter = new PrintWriter(err)) {
+ int exitCode = runMain(outWriter, errWriter, args);
+ return new RunCLI(exitCode, out.toString(), err.toString(), args);
+ }
+ }
+
+ private static int runMain(PrintWriter out, PrintWriter err, String... arguments) {
+ CommandLine commandLine =
+ new CommandLine(new CatalogMigrationCLI())
+ .setExecutionExceptionHandler(
+ (ex, cmd, parseResult) -> {
+ if (enableStacktrace(arguments)) {
+ cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex));
+ } else {
+ cmd.getErr().println("Error during CLI execution: " + ex.getMessage());
+ }
+ return cmd.getExitCodeExceptionMapper() != null
+ ? cmd.getExitCodeExceptionMapper().getExitCode(ex)
+ : cmd.getCommandSpec().exitCodeOnExecutionException();
+ });
+ if (null != out) {
+ commandLine = commandLine.setOut(out);
+ }
+ if (null != err) {
+ commandLine = commandLine.setErr(err);
+ }
+ try {
+ return commandLine.execute(arguments);
+ } finally {
+ commandLine.getErr().flush();
+ }
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public String getOut() {
+ return out;
+ }
+
+ public String getErr() {
+ return err;
+ }
+
+ private static boolean enableStacktrace(String... args) {
+ for (String arg : args) {
+ if (arg.equalsIgnoreCase("--stacktrace")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "org.apache.polaris.catalogs.migrator.cli"
+ + ".RunCLI{args=%s%nexitCode=%d%n%nstdout:%n%s%n%nstderr:%n%s",
+ Arrays.toString(args), exitCode, out, err);
+ }
+}
diff --git a/ide-name.txt b/ide-name.txt
new file mode 100644
index 00000000..beeca0c1
--- /dev/null
+++ b/ide-name.txt
@@ -0,0 +1 @@
+polaris-tools
diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md
new file mode 100644
index 00000000..81e54a37
--- /dev/null
+++ b/polaris-synchronizer/README.md
@@ -0,0 +1,162 @@
+# Objective
+
+To provide users of [Apache Polaris (Incubating)](https://github.com/apache/polaris) a tool to be able to easily and efficiently
+migrate their entities from one Polaris instance to another.
+
+Polaris is a catalog of catalogs. It can become cumbersome to perform catalog-by-catalog migration of each and every catalog contained
+within a Polaris instance. Additionally, while migrating catalog-by-catalog Iceberg entities is achievable using the
+existing generic [iceberg-catalog-migrator](../iceberg-catalog-migrator/README.md), the existing tool will not migrate
+Polaris specific entities, like principal-roles, catalog-roles, grants.
+
+## Use Cases
+* **Migration:** A user may have an active Polaris deployment that they want to migrate to a managed cloud offering like
+ [Snowflake Open Catalog](https://www.snowflake.com/en/product/features/open-catalog/).
+* **Preventing Vendor Lock-In:** A user may currently have a managed Polaris offering and want the freedom to switch providers or to host Polaris themselves.
+* **Mirroring/Disaster Recovery:** Modern data solutions require employing redundancy to ensure no single point of
+ failure. The tool can be scheduled on a cron to run periodic incremental syncs.
+
+The tool currently supports migrating the following Polaris Management entities:
+* Principal roles
+* Catalogs
+* Catalog Roles
+* Assignment of Catalog Roles to Principal Roles
+* Grants
+
+
+> :warning: Polaris principals and their assignments to Principal roles are not supported for migration
+> by this tool. Migrating client credentials stored in Polaris is not possible nor is it secure. Polaris
+> principals must be manually migrated between Polaris instances.
+
+The tool currently supports migrating the following Iceberg entities:
+* Namespaces
+* Tables
+
+# Building the Tool from Source
+
+**Prerequisite:** Must have Java installed in your machine (Java 21 is recommended and the minimum Java version) to use this CLI tool.
+
+```
+gradlew build # build and run tests
+gradlew assemble # build without running tests
+```
+
+The default build location for the built JAR will be `cli/build/libs/`
+
+# Migrating between Polaris Instances
+
+### Step 1: Create a principal with read-only access to catalog internals on the source Polaris instance.
+
+**This step only has to be completed once.**
+
+Polaris is built with a separation between access and metadata management permissions. The `service_admin`
+may have permissions to create access related entities like principal roles, catalog roles, and grants, but may not necessarily
+possess the ability to view Iceberg content of catalogs, like namespaces and tables. We need to create a super user principal
+that has access to all entities on the source Polaris instance in order to migrate them.
+
+To do this, we can use the `create-omnipotent-principal` command to create a principal, principal role,
+and a catalog role per catalog with the appropriate grants to read all entities on the source Polaris instance.
+
+**Example:** Create a **read-only** principal on the source Polaris instance, and replace it if it already exists,
+with 10 concurrent catalog setup threads:
+```
+java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \
+--polaris-client-id root \
+--polaris-client-secret \
+--polaris-base-url http://localhost:8181 \
+--polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \
+--polaris-scope PRINCIPAL_ROLE:ALL \
+--replace \ # replace if it exists
+--concurrency 10 # 10 concurrent catalog setup threads
+```
+
+Upon finishing execution, the tool will output the principal name and client credentials for this
+principal. **Make sure to note these down as they will be necessary for the migration step.**
+
+**Example Output:**
+```
+======================================================
+Omnipotent Principal Credentials:
+name = omnipotent-principal-XXXXX
+clientId = ff7s8f9asbX10
+clientSecret =
+======================================================
+```
+
+Additionally, at the end of execution the command will output a list of catalogs for which catalog setup failed.
+**These catalogs may experience failure during migration**.
+
+**Example Output:**
+```
+Encountered issues creating catalog roles for the following catalogs: [catalog-1, catalog-2]
+```
+
+### Step 2: Create a principal with read-write access to catalog internals on the target Polaris instance.
+
+**This step only has to be completed once.**
+
+The same `create-omnipotent-principal` command can also be used to now create a **read-write** principal on the target
+Polaris instance so that the tool can create entities on the target.
+
+To create a read-write principal, we simply specify the `--write-access` option.
+
+**Example:** Create a read-write principal on your target Polaris instance, replacing it if it exists, with 10 concurrent
+catalog setup threads.
+```
+java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \
+--polaris-client-id root \
+--polaris-client-secret \
+--polaris-base-url http://localhost:8181 \
+--polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \
+--polaris-scope PRINCIPAL_ROLE:ALL \
+--replace \ # replace if it exists
+--concurrency 10 \ # 10 concurrent catalog setup threads
+--write-access # give the principal write access to catalog internals
+```
+
+Similarly to the last step, the tool will output the client credentials and principal name. Again, these need to be noted
+for subsequent steps.
+
+**Example Output:**
+```
+======================================================
+Omnipotent Principal Credentials:
+name = omnipotent-principal-YYYYY
+clientId = 0af20a3a0037a40d
+clientSecret =
+======================================================
+```
+
+> :warning: `service_admin` is not guaranteed to have access management level grants on every catalog. This is usually
+> delegated to the `catalog_admin` role, which is automatically granted to whichever principal role was used to create
+> the catalog. This means that while the tool can detect this catalog when run with `service_admin` level access,
+> it cannot create an omnipotent principal for this catalog. To remedy this, create a catalog-role with `CATALOG_MANAGE_ACCESS`
+> grants for the catalog, and assign it to the principal used to run this tool (presumably, a principal with the `servic_admin`
+> principal role). Then, re-running `create-omnipotent-principal` should be able to create the relevant entities for that catalog.
+
+### Step 3: Running the Migration/Synchronization
+
+Running the synchronization requires minimal reconfiguration, can be run idempotently, and will attempt to only copy over the
+diff between the source and target Polaris instances. This can be achieved using the `sync-polaris` command.
+
+**Example** Running the synchronization between source Polaris instance using an access token, and a target Polaris instance
+using client credentials.
+```
+java -jar polaris-synchronizer-cli.jar sync-polaris \
+--source-base-url http://localhost:8182 \
+--source-access-token \
+--target-base-url http://localhost:8181 \
+--target-client-id root \
+--target-client-secret \
+--target-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \
+--target-scope PRINCIPAL_ROLE:ALL \
+--source-omni-principal-name omnipotent-principal-XXXXX \
+--source-omni-client-id ff7s8f9asbX10 \
+--source-omni-client-secret \
+--target-omni-principal-name omnipotent-principal-YYYYY \
+--target-omni-client-id 0af20a3a0037a40d \
+--target-omni-client-secret
+```
+
+> :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source
+> nor remove or modify them on the target. This is to accommodate that the tool itself will be running with the permission
+> levels for these principals and roles, and we do not want to modify the tool's permissions at runtime.
diff --git a/polaris-synchronizer/api/build.gradle.kts b/polaris-synchronizer/api/build.gradle.kts
new file mode 100644
index 00000000..39691307
--- /dev/null
+++ b/polaris-synchronizer/api/build.gradle.kts
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ `java-library`
+ `maven-publish`
+ signing
+ `build-conventions`
+ alias(libs.plugins.openapi.generator.gradle.plugin)
+}
+
+java {
+ toolchain {
+ languageVersion.set(JavaLanguageVersion.of(21)) // Set the compilation JDK to 21
+ }
+}
+
+dependencies {
+ implementation(libs.openapi.generator)
+ implementation(libs.jakarta.annotation)
+ implementation(libs.iceberg.spark.runtime)
+ implementation(libs.slf4j)
+
+ implementation(libs.hadoop.common) {
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("javax.servlet", "servlet-api")
+ exclude("com.google.code.gson", "gson")
+ exclude("commons-beanutils")
+ }
+
+ testImplementation(platform(libs.junit.bom))
+ testImplementation("org.junit.jupiter:junit-jupiter-params")
+ testImplementation("org.junit.jupiter:junit-jupiter-api")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
+ testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+}
+
+tasks.register(
+ "generatePolarisManagementClient"
+) {
+ inputSpec.set("$projectDir/src/main/resources/polaris-management-service.yml")
+ generatorName.set("java")
+ outputDir.set("${layout.buildDirectory.get()}/generated")
+ apiPackage.set("org.apache.polaris.management.client")
+ modelPackage.set("org.apache.polaris.core.admin.model")
+ removeOperationIdPrefix.set(true)
+
+ globalProperties.set(
+ mapOf(
+ "apis" to "",
+ "models" to "",
+ "supportingFiles" to "",
+ "apiDocs" to "false",
+ "modelTests" to "false",
+ )
+ )
+
+ additionalProperties.set(
+ mapOf(
+ "apiNamePrefix" to "PolarisManagement",
+ "apiNameSuffix" to "Api",
+ "metricsPrefix" to "polaris.management",
+ )
+ )
+
+ configOptions.set(
+ mapOf(
+ "library" to "native",
+ "sourceFolder" to "src/main/java",
+ "useJakartaEe" to "true",
+ "useBeanValidation" to "false",
+ "openApiNullable" to "false",
+ "useRuntimeException" to "true",
+ "supportUrlQuery" to "false",
+ )
+ )
+
+ importMappings.set(
+ mapOf(
+ "AbstractOpenApiSchema" to "org.apache.polaris.core.admin.model.AbstractOpenApiSchema",
+ "AddGrantRequest" to "org.apache.polaris.core.admin.model.AddGrantRequest",
+ "AwsStorageConfigInfo" to "org.apache.polaris.core.admin.model.AwsStorageConfigInfo",
+ "AzureStorageConfigInfo" to "org.apache.polaris.core.admin.model.AzureStorageConfigInfo",
+ "Catalog" to "org.apache.polaris.core.admin.model.Catalog",
+ "CatalogGrant" to "org.apache.polaris.core.admin.model.CatalogGrant",
+ "CatalogPrivilege" to "org.apache.polaris.core.admin.model.CatalogPrivilege",
+ "CatalogProperties" to "org.apache.polaris.core.admin.model.CatalogProperties",
+ "CatalogRole" to "org.apache.polaris.core.admin.model.CatalogRole",
+ "CatalogRoles" to "org.apache.polaris.core.admin.model.CatalogRoles",
+ "Catalogs" to "org.apache.polaris.core.admin.model.Catalogs",
+ "CreateCatalogRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRequest",
+ "CreateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRoleRequest",
+ "CreatePrincipalRequest" to "org.apache.polaris.core.admin.model.CreatePrincipalRequest",
+ "CreatePrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest",
+ "ExternalCatalog" to "org.apache.polaris.core.admin.model.ExternalCatalog",
+ "FileStorageConfigInfo" to "org.apache.polaris.core.admin.model.FileStorageConfigInfo",
+ "GcpStorageConfigInfo" to "org.apache.polaris.core.admin.model.GcpStorageConfigInfo",
+ "GrantCatalogRoleRequest" to "org.apache.polaris.core.admin.model.GrantCatalogRoleRequest",
+ "GrantPrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest",
+ "GrantResource" to "org.apache.polaris.core.admin.model.GrantResource",
+ "GrantResources" to "org.apache.polaris.core.admin.model.GrantResources",
+ "NamespaceGrant" to "org.apache.polaris.core.admin.model.NamespaceGrant",
+ "NamespacePrivilege" to "org.apache.polaris.core.admin.model.NamespacePrivilege",
+ "PolarisCatalog" to "org.apache.polaris.core.admin.model.PolarisCatalog",
+ "Principal" to "org.apache.polaris.core.admin.model.Principal",
+ "PrincipalRole" to "org.apache.polaris.core.admin.model.PrincipalRole",
+ "PrincipalRoles" to "org.apache.polaris.core.admin.model.PrincipalRoles",
+ "PrincipalWithCredentials" to "org.apache.polaris.core.admin.model.PrincipalWithCredentials",
+ "PrincipalWithCredentialsCredentials" to
+ "org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials",
+ "Principals" to "org.apache.polaris.core.admin.model.Principals",
+ "RevokeGrantRequest" to "org.apache.polaris.core.admin.model.RevokeGrantRequest",
+ "StorageConfigInfo" to "org.apache.polaris.core.admin.model.StorageConfigInfo",
+ "TableGrant" to "org.apache.polaris.core.admin.model.TableGrant",
+ "TablePrivilege" to "org.apache.polaris.core.admin.model.TablePrivilege",
+ "UpdateCatalogRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRequest",
+ "UpdateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest",
+ "UpdatePrincipalRequest" to "org.apache.polaris.core.admin.model.UpdatePrincipalRequest",
+ "UpdatePrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest",
+ "ViewGrant" to "org.apache.polaris.core.admin.model.ViewGrant",
+ "ViewPrivilege" to "org.apache.polaris.core.admin.model.ViewPrivilege",
+ )
+ )
+}
+
+tasks.named("compileJava") { dependsOn("generatePolarisManagementClient") }
+
+sourceSets.main { java.srcDir("${layout.buildDirectory.get()}/generated/src/main/java") }
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java
new file mode 100644
index 00000000..18ff23c3
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.http.HttpStatus;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.AddGrantRequest;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.CreateCatalogRequest;
+import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest;
+import org.apache.polaris.core.admin.model.CreatePrincipalRequest;
+import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest;
+import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest;
+import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.core.admin.model.RevokeGrantRequest;
+import org.apache.polaris.management.ApiException;
+import org.apache.polaris.management.client.PolarisManagementDefaultApi;
+import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog;
+
+/**
+ * Service class that wraps Polaris HTTP client and performs recursive operations like drops on
+ * overwrites.
+ */
+public class PolarisService {
+
+ private final PolarisManagementDefaultApi api;
+
+ private final Map catalogProperties;
+
+ public PolarisService(PolarisManagementDefaultApi api, Map catalogProperties) {
+ this.api = api;
+ this.catalogProperties = catalogProperties;
+ }
+
+ public List listPrincipals() {
+ return this.api.listPrincipals().getPrincipals();
+ }
+
+ public Principal getPrincipal(String principalName) {
+ return this.api.getPrincipal(principalName);
+ }
+
+ public boolean principalExists(String principalName) {
+ try {
+ getPrincipal(principalName);
+ return true;
+ } catch (ApiException apiException) {
+ if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) {
+ return false;
+ }
+ throw apiException;
+ }
+ }
+
+ public PrincipalWithCredentials createPrincipal(Principal principal, boolean overwrite) {
+ if (overwrite) {
+ removePrincipal(principal.getName());
+ }
+
+ CreatePrincipalRequest request = new CreatePrincipalRequest().principal(principal);
+ return this.api.createPrincipal(request);
+ }
+
+ public void removePrincipal(String principalName) {
+ this.api.deletePrincipal(principalName);
+ }
+
+ public void assignPrincipalRole(String principalName, String principalRoleName) {
+ GrantPrincipalRoleRequest request =
+ new GrantPrincipalRoleRequest().principalRole(new PrincipalRole().name(principalRoleName));
+ this.api.assignPrincipalRole(principalName, request);
+ }
+
+ public void createPrincipalRole(PrincipalRole principalRole, boolean overwrite) {
+ if (overwrite) {
+ removePrincipalRole(principalRole.getName());
+ }
+ CreatePrincipalRoleRequest request =
+ new CreatePrincipalRoleRequest().principalRole(principalRole);
+ this.api.createPrincipalRole(request);
+ }
+
+ public List listPrincipalRolesAssignedForPrincipal(String principalName) {
+ return this.api.listPrincipalRolesAssigned(principalName).getRoles();
+ }
+
+ public List listPrincipalRoles() {
+ return this.api.listPrincipalRoles().getRoles();
+ }
+
+ public List listAssigneePrincipalRolesForCatalogRole(
+ String catalogName, String catalogRoleName) {
+ return this.api
+ .listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName)
+ .getRoles();
+ }
+
+ public void assignCatalogRoleToPrincipalRole(
+ String principalRoleName, String catalogName, String catalogRoleName) {
+ GrantCatalogRoleRequest request =
+ new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName));
+ this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request);
+ }
+
+ public void removeCatalogRoleFromPrincipalRole(
+ String principalRoleName, String catalogName, String catalogRoleName) {
+ this.api.revokeCatalogRoleFromPrincipalRole(principalRoleName, catalogName, catalogRoleName);
+ }
+
+ public PrincipalRole getPrincipalRole(String principalRoleName) {
+ return this.api.getPrincipalRole(principalRoleName);
+ }
+
+ public boolean principalRoleExists(String principalRoleName) {
+ try {
+ getPrincipalRole(principalRoleName);
+ return true;
+ } catch (ApiException apiException) {
+ if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) {
+ return false;
+ }
+ throw apiException;
+ }
+ }
+
+ public void removePrincipalRole(String principalRoleName) {
+ this.api.deletePrincipalRole(principalRoleName);
+ }
+
+ public List listCatalogs() {
+ return this.api.listCatalogs().getCatalogs();
+ }
+
+ public void createCatalog(Catalog catalog) {
+ CreateCatalogRequest request = new CreateCatalogRequest().catalog(catalog);
+ this.api.createCatalog(request);
+ }
+
+ /**
+ * Performs a cascading drop on the catalog before recreating.
+ *
+ * @param catalog
+ * @param omnipotentPrincipal necessary to initialize an Iceberg catalog to drop catalog internals
+ */
+ public void overwriteCatalog(Catalog catalog, PrincipalWithCredentials omnipotentPrincipal) {
+ removeCatalogCascade(catalog.getName(), omnipotentPrincipal);
+ createCatalog(catalog);
+ }
+
+ /**
+ * Recursively discover all namespaces contained within an Iceberg catalog.
+ *
+ * @param catalog
+ * @return a list of all the namespaces in the catalog
+ */
+ private List discoverAllNamespaces(org.apache.iceberg.catalog.Catalog catalog) {
+ List namespaces = new ArrayList<>();
+ namespaces.add(Namespace.empty());
+
+ if (catalog instanceof SupportsNamespaces namespaceCatalog) {
+ namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, Namespace.empty()));
+ }
+
+ return namespaces;
+ }
+
+ /**
+ * Discover all child namespaces of a given namespace.
+ *
+ * @param namespaceCatalog a catalog that supports nested namespaces
+ * @param namespace the namespace to look under
+ * @return a list of all child namespaces
+ */
+ private List discoverContainedNamespaces(
+ SupportsNamespaces namespaceCatalog, Namespace namespace) {
+ List immediateChildren = namespaceCatalog.listNamespaces(namespace);
+
+ List namespaces = new ArrayList<>();
+
+ for (Namespace ns : immediateChildren) {
+ namespaces.add(ns);
+
+ // discover children of child namespace
+ namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, ns));
+ }
+
+ return namespaces;
+ }
+
+ /**
+ * Perform a cascading drop of a catalog. Removes all namespaces, tables, catalog-roles first.
+ *
+ * @param catalogName
+ * @param omnipotentPrincipal
+ */
+ public void removeCatalogCascade(
+ String catalogName, PrincipalWithCredentials omnipotentPrincipal) {
+ org.apache.iceberg.catalog.Catalog icebergCatalog =
+ initializeCatalog(catalogName, omnipotentPrincipal);
+
+ // find all namespaces in the catalog
+ List namespaces = discoverAllNamespaces(icebergCatalog);
+
+ List tables = new ArrayList<>();
+
+ // find all tables in the catalog
+ for (Namespace ns : namespaces) {
+ if (!ns.isEmpty()) {
+ tables.addAll(icebergCatalog.listTables(ns));
+ }
+ }
+
+ // drop every table in the catalog
+ for (TableIdentifier table : tables) {
+ icebergCatalog.dropTable(table);
+ }
+
+ // drop every namespace in the catalog, note that because we discovered the namespaces
+ // parent-first, we should reverse over the namespaces to ensure that we drop child namespaces
+ // before we drop parent namespaces, as we cannot drop nonempty namespaces
+ for (Namespace ns : namespaces.reversed()) {
+ // NOTE: this is checking if the namespace is not the empty namespace, not if it is empty
+ // in the sense of containing no tables/namespaces
+ if (!ns.isEmpty() && icebergCatalog instanceof SupportsNamespaces namespaceCatalog) {
+ namespaceCatalog.dropNamespace(ns);
+ }
+ }
+
+ List catalogRoles = listCatalogRoles(catalogName);
+
+ // remove catalog roles under catalog
+ for (CatalogRole catalogRole : catalogRoles) {
+ if (catalogRole.getName().equals("catalog_admin")) continue;
+
+ removeCatalogRole(catalogName, catalogRole.getName());
+ }
+
+ this.api.deleteCatalog(catalogName);
+ }
+
+ public List listCatalogRoles(String catalogName) {
+ return this.api.listCatalogRoles(catalogName).getRoles();
+ }
+
+ public CatalogRole getCatalogRole(String catalogName, String catalogRoleName) {
+ return this.api.getCatalogRole(catalogName, catalogRoleName);
+ }
+
+ public boolean catalogRoleExists(String catalogName, String catalogRoleName) {
+ try {
+ getCatalogRole(catalogName, catalogRoleName);
+ return true;
+ } catch (ApiException apiException) {
+ if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) {
+ return false;
+ }
+ throw apiException;
+ }
+ }
+
+ public void assignCatalogRole(
+ String principalRoleName, String catalogName, String catalogRoleName) {
+ GrantCatalogRoleRequest request =
+ new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName));
+ this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request);
+ }
+
+ public void createCatalogRole(String catalogName, CatalogRole catalogRole, boolean overwrite) {
+ if (overwrite) {
+ removeCatalogRole(catalogName, catalogRole.getName());
+ }
+
+ CreateCatalogRoleRequest request = new CreateCatalogRoleRequest().catalogRole(catalogRole);
+ this.api.createCatalogRole(catalogName, request);
+ }
+
+ public void removeCatalogRole(String catalogName, String catalogRoleName) {
+ this.api.deleteCatalogRole(catalogName, catalogRoleName);
+ }
+
+ public List listGrants(String catalogName, String catalogRoleName) {
+ return this.api.listGrantsForCatalogRole(catalogName, catalogRoleName).getGrants();
+ }
+
+ public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) {
+ AddGrantRequest addGrantRequest = new AddGrantRequest().grant(grant);
+ this.api.addGrantToCatalogRole(catalogName, catalogRoleName, addGrantRequest);
+ }
+
+ public void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant) {
+ RevokeGrantRequest revokeGrantRequest = new RevokeGrantRequest().grant(grant);
+ this.api.revokeGrantFromCatalogRole(catalogName, catalogRoleName, false, revokeGrantRequest);
+ }
+
+ public org.apache.iceberg.catalog.Catalog initializeCatalog(
+ String catalogName, PrincipalWithCredentials migratorPrincipal) {
+ Map currentCatalogProperties = new HashMap<>(catalogProperties);
+ currentCatalogProperties.put("warehouse", catalogName);
+
+ String clientId = migratorPrincipal.getCredentials().getClientId();
+ String clientSecret = migratorPrincipal.getCredentials().getClientSecret();
+ currentCatalogProperties.putIfAbsent(
+ "credential", String.format("%s:%s", clientId, clientSecret));
+ currentCatalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL");
+
+ return CatalogUtil.loadCatalog(
+ PolarisCatalog.class.getName(), "SOURCE_CATALOG_REST", currentCatalogProperties, null);
+ }
+
+ /**
+ * Perform cascading drop of a namespace.
+ *
+ * @param icebergCatalog the iceberg catalog to use
+ * @param namespace the namespace to drop
+ */
+ public void dropNamespaceCascade(
+ org.apache.iceberg.catalog.Catalog icebergCatalog, Namespace namespace) {
+ if (icebergCatalog instanceof SupportsNamespaces namespaceCatalog) {
+ List namespaces = discoverContainedNamespaces(namespaceCatalog, namespace);
+
+ List tables = new ArrayList<>();
+
+ for (Namespace ns : namespaces) {
+ tables.addAll(icebergCatalog.listTables(ns));
+ }
+
+ tables.addAll(icebergCatalog.listTables(namespace));
+
+ for (TableIdentifier table : tables) {
+ icebergCatalog.dropTable(table);
+ }
+
+ // go over in reverse order of namespaces since we discover namespaces
+ // in the parent -> child order, so we need to drop all children
+ // before we can drop the parent
+ for (Namespace ns : namespaces.reversed()) {
+ namespaceCatalog.dropNamespace(ns);
+ }
+
+ namespaceCatalog.dropNamespace(namespace);
+ }
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java
new file mode 100644
index 00000000..2dacd37a
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.http.HttpHeaders;
+import org.apache.polaris.management.ApiClient;
+import org.apache.polaris.management.client.PolarisManagementDefaultApi;
+import org.apache.polaris.tools.sync.polaris.http.OAuth2Util;
+
+/** Used to initialize a {@link PolarisService}. */
+public class PolarisServiceFactory {
+
+ private static void validatePolarisInstanceProperties(
+ String baseUrl,
+ String accessToken,
+ String oauth2ServerUri,
+ String clientId,
+ String clientSecret,
+ String scope) {
+ if (baseUrl == null) {
+ throw new IllegalArgumentException("baseUrl is required but was not provided");
+ }
+
+ if (accessToken != null) {
+ return;
+ }
+
+ final String oauthErrorMessage =
+ "Either the accessToken property must be provided, or all of oauth2ServerUri, clientId, clientSecret, scope";
+
+ if (oauth2ServerUri == null || clientId == null || clientSecret == null || scope == null) {
+ throw new IllegalArgumentException(oauthErrorMessage);
+ }
+ }
+
+ public static PolarisService newPolarisService(
+ String baseUrl, String oauth2ServerUri, String clientId, String clientSecret, String scope)
+ throws IOException {
+ validatePolarisInstanceProperties(
+ baseUrl, null, oauth2ServerUri, clientId, clientSecret, scope);
+
+ String accessToken = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope);
+
+ return newPolarisService(baseUrl, accessToken);
+ }
+
+ public static PolarisService newPolarisService(String baseUrl, String accessToken) {
+ validatePolarisInstanceProperties(baseUrl, accessToken, null, null, null, null);
+
+ ApiClient client = new ApiClient();
+ client.updateBaseUri(baseUrl + "/api/management/v1");
+
+ // TODO: Add token refresh
+ client.setRequestInterceptor(
+ requestBuilder -> {
+ requestBuilder.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken);
+ });
+
+ Map catalogProperties = new HashMap<>();
+ catalogProperties.putIfAbsent("uri", baseUrl + "/api/catalog");
+
+ PolarisManagementDefaultApi polarisClient = new PolarisManagementDefaultApi(client);
+ return new PolarisService(polarisClient, catalogProperties);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java
new file mode 100644
index 00000000..90b4174c
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java
@@ -0,0 +1,1135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.access.AccessControlService;
+import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag;
+import org.apache.polaris.tools.sync.polaris.catalog.ETagService;
+import org.apache.polaris.tools.sync.polaris.catalog.NotModifiedException;
+import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog;
+import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Encapsulates idempotent and failure-safe logic to perform Polaris entity syncs. Performs logging
+ * with configurability and all actions related to the generated sync plans.
+ */
+public class PolarisSynchronizer {
+
+ private final Logger clientLogger;
+
+ private final SynchronizationPlanner syncPlanner;
+
+ private final PolarisService source;
+
+ private final PolarisService target;
+
+ private final PrincipalWithCredentials sourceOmnipotentPrincipal;
+
+ private final PrincipalWithCredentials targetOmnipotentPrincipal;
+
+ private final PrincipalRole sourceOmnipotentPrincipalRole;
+
+ private final PrincipalRole targetOmnipotentPrincipalRole;
+
+ private final AccessControlService sourceAccessControlService;
+
+ private final AccessControlService targetAccessControlService;
+
+ private final ETagService etagService;
+
+ public PolarisSynchronizer(
+ Logger clientLogger,
+ SynchronizationPlanner synchronizationPlanner,
+ PrincipalWithCredentials sourceOmnipotentPrincipal,
+ PrincipalWithCredentials targetOmnipotentPrincipal,
+ PolarisService source,
+ PolarisService target,
+ ETagService etagService) {
+ this.clientLogger =
+ clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger;
+ this.syncPlanner = synchronizationPlanner;
+ this.sourceOmnipotentPrincipal = sourceOmnipotentPrincipal;
+ this.targetOmnipotentPrincipal = targetOmnipotentPrincipal;
+ this.source = source;
+ this.target = target;
+ this.sourceAccessControlService = new AccessControlService(source);
+ this.targetAccessControlService = new AccessControlService(target);
+
+ this.sourceOmnipotentPrincipalRole =
+ sourceAccessControlService.getOmnipotentPrincipalRoleForPrincipal(
+ sourceOmnipotentPrincipal.getPrincipal().getName());
+ this.targetOmnipotentPrincipalRole =
+ targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal(
+ targetOmnipotentPrincipal.getPrincipal().getName());
+ this.etagService = etagService;
+ }
+
+ /**
+ * Calculates the total number of sync tasks to complete.
+ *
+ * @param plan the plan to scan for cahnges
+ * @return the nuber of syncs to perform
+ */
+ private int totalSyncsToComplete(SynchronizationPlan plan) {
+ return plan.entitiesToCreate().size()
+ + plan.entitiesToOverwrite().size()
+ + plan.entitiesToRemove().size();
+ }
+
+ /** Sync principal roles from source to target. */
+ public void syncPrincipalRoles() {
+ List principalRolesSource;
+
+ try {
+ principalRolesSource = source.listPrincipalRoles();
+ clientLogger.info("Listed {} principal-roles from source.", principalRolesSource.size());
+ } catch (Exception e) {
+ clientLogger.error("Failed to list principal-roles from source.", e);
+ return;
+ }
+
+ List principalRolesTarget;
+
+ try {
+ principalRolesTarget = target.listPrincipalRoles();
+ clientLogger.info("Listed {} principal-roles from target.", principalRolesTarget.size());
+ } catch (Exception e) {
+ clientLogger.error("Failed to list principal-roles from target.", e);
+ return;
+ }
+
+ SynchronizationPlan principalRoleSyncPlan =
+ syncPlanner.planPrincipalRoleSync(principalRolesSource, principalRolesTarget);
+
+ principalRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ principalRole ->
+ clientLogger.info("Skipping principal-role {}.", principalRole.getName()));
+
+ principalRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "No change detected for principal-role {}, skipping.",
+ principalRole.getName()));
+
+ int syncsCompleted = 0;
+ final int totalSyncsToComplete = totalSyncsToComplete(principalRoleSyncPlan);
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.createPrincipalRole(principalRole, false);
+ clientLogger.info(
+ "Created principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to create principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.createPrincipalRole(principalRole, true);
+ clientLogger.info(
+ "Overwrote principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to overwrite principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.removePrincipalRole(principalRole.getName());
+ clientLogger.info(
+ "Removed principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to remove principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /**
+ * Sync assignments of principal roles to a catalog role.
+ *
+ * @param catalogName the catalog that the catalog role is in
+ * @param catalogRoleName the name of the catalog role
+ */
+ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName) {
+ List principalRolesSource;
+
+ try {
+ principalRolesSource =
+ source.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} assignee principal-roles for catalog-role {} in catalog {} from source.",
+ principalRolesSource.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list assignee principal-roles for catalog-role {} in catalog {} from source.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ List principalRolesTarget;
+
+ try {
+ principalRolesTarget =
+ target.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} assignee principal-roles for catalog-role {} in catalog {} from target.",
+ principalRolesTarget.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list assignee principal-roles for catalog-role {} in catalog {} from target.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan assignedPrincipalRoleSyncPlan =
+ syncPlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ catalogName, catalogRoleName, principalRolesSource, principalRolesTarget);
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "Skipping assignment of principal-role {} to catalog-role {} in catalog {}.",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName));
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "Principal-role {} is already assigned to catalog-role {} in catalog {}. Skipping.",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(assignedPrincipalRoleSyncPlan);
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.assignCatalogRoleToPrincipalRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.assignCatalogRoleToPrincipalRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.removeCatalogRoleFromPrincipalRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Revoked principal-role {} from catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to revoke principal-role {} from catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /** Sync catalogs across the source and target polaris instance. */
+ public void syncCatalogs() {
+ List catalogsSource;
+
+ try {
+ catalogsSource = source.listCatalogs();
+ clientLogger.info("Listed {} catalogs from source.", catalogsSource.size());
+ } catch (Exception e) {
+ clientLogger.error("Failed to list catalogs from source.", e);
+ return;
+ }
+
+ List catalogsTarget;
+
+ try {
+ catalogsTarget = target.listCatalogs();
+ clientLogger.info("Listed {} catalogs from target.", catalogsTarget.size());
+ } catch (Exception e) {
+ clientLogger.error("Failed to list catalogs from target.", e);
+ return;
+ }
+
+ SynchronizationPlan catalogSyncPlan =
+ syncPlanner.planCatalogSync(catalogsSource, catalogsTarget);
+
+ catalogSyncPlan
+ .entitiesToSkip()
+ .forEach(catalog -> clientLogger.info("Skipping catalog {}.", catalog.getName()));
+
+ catalogSyncPlan
+ .entitiesToSkipAndSkipChildren()
+ .forEach(
+ catalog ->
+ clientLogger.info(
+ "Skipping catalog {} and all child entities.", catalog.getName()));
+
+ catalogSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ catalog ->
+ clientLogger.info(
+ "No change detected in catalog {}. Skipping.", catalog.getName()));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(catalogSyncPlan);
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToCreate()) {
+ try {
+ target.createCatalog(catalog);
+ clientLogger.info(
+ "Created catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to create catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToOverwrite()) {
+ try {
+ setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName());
+ target.overwriteCatalog(catalog, targetOmnipotentPrincipal);
+ clientLogger.info(
+ "Overwrote catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to overwrite catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToRemove()) {
+ try {
+ setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName());
+ target.removeCatalogCascade(catalog.getName(), targetOmnipotentPrincipal);
+ clientLogger.info(
+ "Removed catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to remove catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToSyncChildren()) {
+ syncCatalogRoles(catalog.getName());
+
+ org.apache.iceberg.catalog.Catalog sourceIcebergCatalog;
+
+ try {
+ sourceIcebergCatalog = initializeIcebergCatalogSource(catalog.getName());
+ clientLogger.info(
+ "Initialized Iceberg REST catalog for Polaris catalog {} on source.",
+ catalog.getName());
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to initialize Iceberg REST catalog for Polaris catalog {} on source.",
+ catalog.getName(),
+ e);
+ continue;
+ }
+
+ org.apache.iceberg.catalog.Catalog targetIcebergCatalog;
+
+ try {
+ targetIcebergCatalog = initializeIcebergCatalogTarget(catalog.getName());
+ clientLogger.info(
+ "Initialized Iceberg REST catalog for Polaris catalog {} on target.",
+ catalog.getName());
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to initialize Iceberg REST catalog for Polaris catalog {} on target.",
+ catalog.getName(),
+ e);
+ continue;
+ }
+
+ syncNamespaces(
+ catalog.getName(), Namespace.empty(), sourceIcebergCatalog, targetIcebergCatalog);
+ }
+ }
+
+ /**
+ * Sync catalog roles across the source and polaris instance for a catalog.
+ *
+ * @param catalogName the catalog to sync roles for
+ */
+ public void syncCatalogRoles(String catalogName) {
+ List catalogRolesSource;
+
+ try {
+ catalogRolesSource = source.listCatalogRoles(catalogName);
+ clientLogger.info(
+ "Listed {} catalog-roles for catalog {} from source.",
+ catalogRolesSource.size(),
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list catalog-roles for catalog {} from source.", catalogName, e);
+ return;
+ }
+
+ List catalogRolesTarget;
+
+ try {
+ catalogRolesTarget = target.listCatalogRoles(catalogName);
+ clientLogger.info(
+ "Listed {} catalog-roles for catalog {} from target.",
+ catalogRolesTarget.size(),
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list catalog-roles for catalog {} from target.", catalogName, e);
+ return;
+ }
+
+ SynchronizationPlan catalogRoleSyncPlan =
+ syncPlanner.planCatalogRoleSync(catalogName, catalogRolesSource, catalogRolesTarget);
+
+ catalogRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "Skipping catalog-role {} in catalog {}.", catalogRole.getName(), catalogName));
+
+ catalogRoleSyncPlan
+ .entitiesToSkipAndSkipChildren()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "Skipping catalog-role {} in catalog {} and all child entities.",
+ catalogRole.getName(),
+ catalogName));
+
+ catalogRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "No change detected in catalog-role {} in catalog {}. Skipping.",
+ catalogRole.getName(),
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(catalogRoleSyncPlan);
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.createCatalogRole(catalogName, catalogRole, false);
+ clientLogger.info(
+ "Created catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to create catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.createCatalogRole(catalogName, catalogRole, true);
+ clientLogger.info(
+ "Overwrote catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to overwrite catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.removeCatalogRole(catalogName, catalogRole.getName());
+ clientLogger.info(
+ "Removed catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to remove catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToSyncChildren()) {
+ syncAssigneePrincipalRolesForCatalogRole(catalogName, catalogRole.getName());
+ syncGrants(catalogName, catalogRole.getName());
+ }
+ }
+
+ /**
+ * Sync grants for a catalog role across the source and the target.
+ *
+ * @param catalogName
+ * @param catalogRoleName
+ */
+ private void syncGrants(String catalogName, String catalogRoleName) {
+ List grantsSource;
+
+ try {
+ grantsSource = source.listGrants(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} grants for catalog-role {} in catalog {} from source.",
+ grantsSource.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list grants for catalog-role {} in catalog {} from source.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ List grantsTarget;
+
+ try {
+ grantsTarget = target.listGrants(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} grants for catalog-role {} in catalog {} from target.",
+ grantsTarget.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list grants for catalog-role {} in catalog {} from target.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan grantSyncPlan =
+ syncPlanner.planGrantSync(catalogName, catalogRoleName, grantsSource, grantsTarget);
+
+ grantSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ grant ->
+ clientLogger.info(
+ "Skipping addition of grant {} to catalog-role {} in catalog {}.",
+ grant.getType(),
+ catalogRoleName,
+ catalogName));
+
+ grantSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ grant ->
+ clientLogger.info(
+ "Grant {} was already added to catalog-role {} in catalog {}. Skipping.",
+ grant.getType(),
+ catalogRoleName,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(grantSyncPlan);
+
+ for (GrantResource grant : grantSyncPlan.entitiesToCreate()) {
+ try {
+ target.addGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Added grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (GrantResource grant : grantSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.addGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Added grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (GrantResource grant : grantSyncPlan.entitiesToRemove()) {
+ try {
+ target.revokeGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Revoked grant {} from catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to revoke grant {} from catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /**
+ * Setup an omnipotent principal for the provided catalog on the source Polaris instance.
+ *
+ * @param catalogName
+ */
+ private void setupOmnipotentCatalogRoleIfNotExistsTarget(String catalogName) {
+ if (!this.targetAccessControlService.omnipotentCatalogRoleExists(catalogName)) {
+ clientLogger.info(
+ "No omnipotent catalog-role exists for catalog {} on target. Going to set one up.",
+ catalogName);
+
+ targetAccessControlService.setupOmnipotentRoleForCatalog(
+ catalogName, targetOmnipotentPrincipalRole, false, true);
+
+ clientLogger.info("Setup omnipotent catalog-role for catalog {} on target.", catalogName);
+ }
+ }
+
+ /**
+ * Setup an omnipotent principal for the provided catalog on the target Polaris instance.
+ *
+ * @param catalogName
+ */
+ private void setupOmnipotentCatalogRoleIfNotExistsSource(String catalogName) {
+ if (!this.sourceAccessControlService.omnipotentCatalogRoleExists(catalogName)) {
+ clientLogger.info(
+ "No omnipotent catalog-role exists for catalog {} on source. Going to set one up.",
+ catalogName);
+
+ sourceAccessControlService.setupOmnipotentRoleForCatalog(
+ catalogName, sourceOmnipotentPrincipalRole, false, false);
+
+ clientLogger.info("Setup omnipotent catalog-role for catalog {} on source.", catalogName);
+ }
+ }
+
+ public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogSource(String catalogName) {
+ setupOmnipotentCatalogRoleIfNotExistsSource(catalogName);
+ return source.initializeCatalog(catalogName, sourceOmnipotentPrincipal);
+ }
+
+ public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogTarget(String catalogName) {
+ setupOmnipotentCatalogRoleIfNotExistsTarget(catalogName);
+ return target.initializeCatalog(catalogName, targetOmnipotentPrincipal);
+ }
+
+ /**
+ * Sync namespaces contained within a parent namespace.
+ *
+ * @param catalogName
+ * @param parentNamespace
+ * @param sourceIcebergCatalog
+ * @param targetIcebergCatalog
+ */
+ public void syncNamespaces(
+ String catalogName,
+ Namespace parentNamespace,
+ org.apache.iceberg.catalog.Catalog sourceIcebergCatalog,
+ org.apache.iceberg.catalog.Catalog targetIcebergCatalog) {
+ // no namespaces to sync if catalog does not implement SupportsNamespaces
+ if (sourceIcebergCatalog instanceof SupportsNamespaces sourceNamespaceCatalog
+ && targetIcebergCatalog instanceof SupportsNamespaces targetNamespaceCatalog) {
+ List namespacesSource;
+
+ try {
+ namespacesSource = sourceNamespaceCatalog.listNamespaces(parentNamespace);
+ clientLogger.info(
+ "Listed {} namespaces in namespace {} for catalog {} from source.",
+ namespacesSource.size(),
+ parentNamespace,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list namespaces in namespace {} for catalog {} from source.",
+ parentNamespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ List namespacesTarget;
+
+ try {
+ namespacesTarget = targetNamespaceCatalog.listNamespaces(parentNamespace);
+ clientLogger.info(
+ "Listed {} namespaces in namespace {} for catalog {} from target.",
+ namespacesTarget.size(),
+ parentNamespace,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list namespaces in namespace {} for catalog {} from target.",
+ parentNamespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan namespaceSynchronizationPlan =
+ syncPlanner.planNamespaceSync(
+ catalogName, parentNamespace, namespacesSource, namespacesTarget);
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(namespaceSynchronizationPlan);
+
+ namespaceSynchronizationPlan
+ .entitiesNotModified()
+ .forEach(
+ namespace ->
+ clientLogger.info(
+ "No change detected for namespace {} in namespace {} for catalog {}, skipping.",
+ namespace,
+ parentNamespace,
+ catalogName));
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToCreate()) {
+ try {
+ targetNamespaceCatalog.createNamespace(namespace);
+ clientLogger.info(
+ "Created namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to create namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToOverwrite()) {
+ try {
+ Map sourceNamespaceMetadata =
+ sourceNamespaceCatalog.loadNamespaceMetadata(namespace);
+ Map targetNamespaceMetadata =
+ targetNamespaceCatalog.loadNamespaceMetadata(namespace);
+
+ if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) {
+ clientLogger.info(
+ "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ continue;
+ }
+
+ target.dropNamespaceCascade(targetIcebergCatalog, namespace);
+ targetNamespaceCatalog.createNamespace(namespace, sourceNamespaceMetadata);
+
+ clientLogger.info(
+ "Overwrote namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to overwrite namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToRemove()) {
+ try {
+ target.dropNamespaceCascade(targetIcebergCatalog, namespace);
+ clientLogger.info(
+ "Removed namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToSyncChildren()) {
+ syncTables(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog);
+ syncNamespaces(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog);
+ }
+ }
+ }
+
+ /**
+ * Sync tables contained within a namespace.
+ *
+ * @param catalogName
+ * @param namespace
+ * @param sourceIcebergCatalog
+ * @param targetIcebergCatalog
+ */
+ public void syncTables(
+ String catalogName,
+ Namespace namespace,
+ org.apache.iceberg.catalog.Catalog sourceIcebergCatalog,
+ org.apache.iceberg.catalog.Catalog targetIcebergCatalog) {
+ Set sourceTables;
+
+ try {
+ sourceTables = new HashSet<>(sourceIcebergCatalog.listTables(namespace));
+ clientLogger.info(
+ "Listed {} tables in namespace {} for catalog {} on source.",
+ sourceTables.size(),
+ namespace,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list tables in namespace {} for catalog {} on source.",
+ namespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ Set targetTables;
+
+ try {
+ targetTables = new HashSet<>(targetIcebergCatalog.listTables(namespace));
+ clientLogger.info(
+ "Listed {} tables in namespace {} for catalog {} on target.",
+ targetTables.size(),
+ namespace,
+ catalogName);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to list tables in namespace {} for catalog {} on target.",
+ namespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan tableSyncPlan =
+ syncPlanner.planTableSync(catalogName, namespace, sourceTables, targetTables);
+
+ tableSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ tableId ->
+ clientLogger.info(
+ "Skipping table {} in namespace {} in catalog {}.",
+ tableId,
+ namespace,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(tableSyncPlan);
+
+ for (TableIdentifier tableId : tableSyncPlan.entitiesToCreate()) {
+ try {
+ Table table = sourceIcebergCatalog.loadTable(tableId);
+
+ if (table instanceof BaseTable baseTable) {
+ targetIcebergCatalog.registerTable(
+ tableId, baseTable.operations().current().metadataFileLocation());
+ } else {
+ throw new IllegalStateException("Cannot register table that does not extend BaseTable.");
+ }
+
+ if (table instanceof BaseTableWithETag tableWithETag) {
+ etagService.storeETag(catalogName, tableId, tableWithETag.etag());
+ }
+
+ clientLogger.info(
+ "Registered table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to register table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (TableIdentifier tableId : tableSyncPlan.entitiesToOverwrite()) {
+ try {
+ Table table;
+
+ if (sourceIcebergCatalog instanceof PolarisCatalog polarisCatalog) {
+ String etag = etagService.getETag(catalogName, tableId);
+ table = polarisCatalog.loadTable(tableId, etag);
+ } else {
+ table = sourceIcebergCatalog.loadTable(tableId);
+ }
+
+ if (table instanceof BaseTable baseTable) {
+ targetIcebergCatalog.dropTable(tableId, /* purge */ false);
+ targetIcebergCatalog.registerTable(
+ tableId, baseTable.operations().current().metadataFileLocation());
+ } else {
+ throw new IllegalStateException("Cannot register table that does not extend BaseTable.");
+ }
+
+ if (table instanceof BaseTableWithETag tableWithETag) {
+ etagService.storeETag(catalogName, tableId, tableWithETag.etag());
+ }
+
+ clientLogger.info(
+ "Dropped and re-registered table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (NotModifiedException e) {
+ clientLogger.info(
+ "Table {} in namespace {} in catalog {} with was not modified, not overwriting in target catalog. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.error(
+ "Failed to drop and re-register table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (TableIdentifier table : tableSyncPlan.entitiesToRemove()) {
+ try {
+ targetIcebergCatalog.dropTable(table, /* purge */ false);
+ clientLogger.info(
+ "Dropped table {} in namespace {} in catalog {}. - {}/{}",
+ table,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ clientLogger.info(
+ "Failed to drop table {} in namespace {} in catalog {}. - {}/{}",
+ table,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java
new file mode 100644
index 00000000..58be6a95
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.access;
+
+public class AccessControlConstants {
+
+ public static final String OMNIPOTENCE_PROPERTY = "IS_OMNIPOTENT_PRINCIPAL";
+
+ protected static final String OMNIPOTENT_PRINCIPAL_NAME_PREFIX = "omnipotent-principal-";
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java
new file mode 100644
index 00000000..c0b4fe6e
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.access;
+
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_MANAGE_METADATA;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_READ_PROPERTIES;
+import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENCE_PROPERTY;
+import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENT_PRINCIPAL_NAME_PREFIX;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.polaris.core.admin.model.CatalogGrant;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.PolarisService;
+
+/**
+ * Service class to facilitate the access control needs of the synchronization. This involves
+ * setting up principals, principal roles, catalog roles, and grants to allow the tool to be able to
+ * introspect into catalog internals like catalog-roles, tables, grants.
+ */
+public class AccessControlService {
+
+ private final PolarisService polaris;
+
+ public AccessControlService(PolarisService polaris) {
+ this.polaris = polaris;
+ }
+
+ /**
+ * Creates or replaces the existing omnipotent principal on the provided polaris instance.
+ *
+ * @param replace if true, if an existing omnipotent principal role exists, it will be dropped and
+ * recreated
+ * @return the principal and credentials for the omnipotent principal
+ */
+ public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) {
+ List principals = polaris.listPrincipals();
+
+ Principal omnipotentPrincipalPrototype =
+ new Principal()
+ .name(OMNIPOTENT_PRINCIPAL_NAME_PREFIX + System.currentTimeMillis())
+ .putPropertiesItem(
+ OMNIPOTENCE_PROPERTY, ""); // this property identifies the omnipotent principal
+
+ for (Principal principal : principals) {
+ if (principal.getProperties() != null
+ && principal.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ if (replace) {
+ // drop existing omnipotent principal in preparation for replacement
+ polaris.removePrincipal(principal.getName());
+ } else {
+ // we cannot create another omnipotent principal and cannot replace the existing, fail
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent principal, but omnipotent "
+ + "principal with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ // existing principal with identifying property does not exist, create a new one
+ return polaris.createPrincipal(omnipotentPrincipalPrototype, false);
+ }
+
+ /**
+ * Retrieves the omnipotent principal role for the provided principalName.
+ *
+ * @param principalName the principal name to search for roles with
+ * @return the principal role for the provided principal, if exists
+ */
+ public PrincipalRole getOmnipotentPrincipalRoleForPrincipal(String principalName) {
+ List principalRolesAssigned =
+ polaris.listPrincipalRolesAssignedForPrincipal(principalName);
+
+ return principalRolesAssigned.stream()
+ .filter(
+ principalRole ->
+ principalRole.getProperties() != null
+ && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new NoSuchElementException(
+ "No omnipotent principal role with property "
+ + OMNIPOTENCE_PROPERTY
+ + " exists for principal "
+ + principalName));
+ }
+
+ /**
+ * Creates a principal role for the omnipotent principal and assigns it to the provided omnipotent
+ * principal.
+ *
+ * @param omnipotentPrincipal the principal to create and assign the role for
+ * @param replace if true, drops existing omnipotent principal roles if they exist before creating
+ * the new one
+ * @return the principal role for the omnipotent principal
+ */
+ public PrincipalRole createAndAssignPrincipalRole(
+ PrincipalWithCredentials omnipotentPrincipal, boolean replace) {
+ List principalRoles = polaris.listPrincipalRoles();
+
+ PrincipalRole omnipotentPrincipalRole =
+ new PrincipalRole()
+ .name(omnipotentPrincipal.getPrincipal().getName())
+ .putPropertiesItem(OMNIPOTENCE_PROPERTY, "");
+
+ for (PrincipalRole principalRole : principalRoles) {
+ if (principalRole.getProperties() != null
+ && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ // replace existing principal role if exists
+ if (replace) {
+ polaris.removePrincipalRole(principalRole.getName());
+ } else {
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent principal role, but omnipotent "
+ + "principal role with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ polaris.createPrincipalRole(omnipotentPrincipalRole, false);
+ polaris.assignPrincipalRole(
+ omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName());
+ return omnipotentPrincipalRole;
+ }
+
+ /**
+ * Creates an omnipotent catalog role for a catalog and assigns it to the provided omnipotent
+ * principal role.
+ *
+ * @param catalogName the catalog to create the catalog role for
+ * @param omnipotentPrincipalRole the omnipotent principal role to assign the created catalog role
+ * to
+ * @param replace if true, drops and recreates the existing omnipotent catalog role
+ * @return the created omnipotent catalog role
+ */
+ public CatalogRole createAndAssignCatalogRole(
+ String catalogName, PrincipalRole omnipotentPrincipalRole, boolean replace) {
+ List catalogRoles = polaris.listCatalogRoles(catalogName);
+
+ for (CatalogRole catalogRole : catalogRoles) {
+ if (catalogRole.getProperties() != null
+ && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ if (replace) {
+ polaris.removeCatalogRole(catalogName, catalogRole.getName());
+ } else {
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent catalog role for catalog "
+ + catalogName
+ + ", but omnipotent principal with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ CatalogRole omnipotentCatalogRole =
+ new CatalogRole()
+ .name(omnipotentPrincipalRole.getName())
+ .putPropertiesItem(OMNIPOTENCE_PROPERTY, "");
+
+ polaris.createCatalogRole(catalogName, omnipotentCatalogRole, false /* overwrite */);
+ polaris.assignCatalogRole(
+ omnipotentPrincipalRole.getName(), catalogName, omnipotentCatalogRole.getName());
+ return omnipotentCatalogRole;
+ }
+
+ /**
+ * Adds grants for privilege level desired on the omnipotent catalog role.
+ *
+ * @param catalogName the catalog to identify the role in
+ * @param catalogRoleName the name of the catalog role to assign the grants tpo
+ * @param withWriteAccess if the catalog role should be given write access to the catalog
+ * internals
+ * @return the grants that were added to the catalog role
+ */
+ public List addGrantsToCatalogRole(
+ String catalogName, String catalogRoleName, boolean withWriteAccess) {
+ if (withWriteAccess) {
+ // write access only requires CATALOG_MANAGE_METADATA
+ CatalogGrant catalogManageMetadata =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(CATALOG_MANAGE_METADATA);
+
+ polaris.addGrant(catalogName, catalogRoleName, catalogManageMetadata);
+ return List.of(catalogManageMetadata);
+ } else {
+ // read access requires reading properties and listing entities for each entity type
+ CatalogGrant catalogReadProperties =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(CATALOG_READ_PROPERTIES);
+
+ CatalogGrant namespaceReadProperties =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(NAMESPACE_READ_PROPERTIES);
+
+ CatalogGrant namespaceList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(NAMESPACE_LIST);
+
+ CatalogGrant tableReadProperties =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_READ_PROPERTIES);
+
+ CatalogGrant tableList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_LIST);
+
+ CatalogGrant viewReadProperties =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_READ_PROPERTIES);
+
+ CatalogGrant viewList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_LIST);
+
+ polaris.addGrant(catalogName, catalogRoleName, catalogReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, namespaceReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, namespaceList);
+ polaris.addGrant(catalogName, catalogRoleName, tableReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, tableList);
+ polaris.addGrant(catalogName, catalogRoleName, viewReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, viewList);
+ return List.of(
+ catalogReadProperties, namespaceReadProperties, tableReadProperties, viewReadProperties);
+ }
+ }
+
+ /**
+ * Determines if an omnipotent catalog role already exists for this catalog.
+ *
+ * @param catalogName the catalog to search in
+ * @return true if exists, false otherwise
+ */
+ public boolean omnipotentCatalogRoleExists(String catalogName) {
+ List catalogRoles = polaris.listCatalogRoles(catalogName);
+
+ return catalogRoles.stream()
+ .anyMatch(
+ catalogRole ->
+ catalogRole.getProperties() != null
+ && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY));
+ }
+
+ /**
+ * Creates catalog role for catalog, assigns it to provided principal role, and assigns grants
+ * with appropriate privilege level.
+ *
+ * @param catalogName the catalog to create the role for
+ * @param omnipotentPrincipalRole the principal role to assign the catalog role to
+ * @param replace if true, drops the existing catalog role if it exists
+ * @param withWriteAccess gives write access to the catalog role
+ */
+ public void setupOmnipotentRoleForCatalog(
+ String catalogName,
+ PrincipalRole omnipotentPrincipalRole,
+ boolean replace,
+ boolean withWriteAccess) {
+ CatalogRole omniPotentCatalogRole =
+ createAndAssignCatalogRole(catalogName, omnipotentPrincipalRole, replace);
+ addGrantsToCatalogRole(catalogName, omniPotentCatalogRole.getName(), withWriteAccess);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java
new file mode 100644
index 00000000..1f28a031
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.metrics.MetricsReporter;
+
+/** Wrapper around {@link BaseTable} that contains the latest ETag for the table. */
+public class BaseTableWithETag extends BaseTable {
+
+ private final String etag;
+
+ public BaseTableWithETag(TableOperations ops, String name, String etag) {
+ super(ops, name);
+ this.etag = etag;
+ }
+
+ public BaseTableWithETag(
+ TableOperations ops, String name, MetricsReporter reporter, String etag) {
+ super(ops, name, reporter);
+ this.etag = etag;
+ }
+
+ public String etag() {
+ return etag;
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java
new file mode 100644
index 00000000..b06e4281
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+/**
+ * Generic interface to provide and store ETags for tables within catalogs. This allows the storage
+ * of the ETag to be completely independent from the tool.
+ */
+public interface ETagService {
+
+ /**
+ * Retrieves the ETag for the table.
+ *
+ * @param catalogName the catalog the table is in
+ * @param tableIdentifier the table identifier
+ * @return The ETag for the last known metadata for the table
+ */
+ String getETag(String catalogName, TableIdentifier tableIdentifier);
+
+ /**
+ * After table loading, stores the fetched ETag.
+ *
+ * @param catalogName the catalog the table is in
+ * @param tableIdentifier the table identifier
+ * @param etag the ETag that was provided by the Iceberg REST api
+ */
+ void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag);
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java
new file mode 100644
index 00000000..a5331554
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import java.util.NoSuchElementException;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.LocationProvider;
+
+/**
+ * Wrapper table operationw class that just allows fetching a provided table metadata. Used to build
+ * a {@link org.apache.iceberg.BaseTable} without having to expose a full-fledged operations class.
+ */
+public class MetadataWrapperTableOperations implements TableOperations {
+
+ private final TableMetadata tableMetadata;
+
+ public MetadataWrapperTableOperations(TableMetadata tableMetadata) {
+ this.tableMetadata = tableMetadata;
+ }
+
+ @Override
+ public TableMetadata current() {
+ return this.tableMetadata;
+ }
+
+ @Override
+ public TableMetadata refresh() {
+ return this.tableMetadata;
+ }
+
+ @Override
+ public void commit(TableMetadata tableMetadata, TableMetadata tableMetadata1) {
+ throw new UnsupportedOperationException("Cannot perform commit.");
+ }
+
+ @Override
+ public FileIO io() {
+ throw new NoSuchElementException("Does not possess file io.");
+ }
+
+ @Override
+ public String metadataFileLocation(String s) {
+ return this.tableMetadata.metadataFileLocation();
+ }
+
+ @Override
+ public LocationProvider locationProvider() {
+ throw new NoSuchElementException("Does not possess location provider.");
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java
new file mode 100644
index 00000000..04f3bd0c
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+/** Implementation that returns nothing and stores no ETags. */
+public class NoOpETagService implements ETagService {
+
+ @Override
+ public String getETag(String catalogName, TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ public void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag) {}
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java
new file mode 100644
index 00000000..6903c642
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+public class NotModifiedException extends RuntimeException {
+
+ public NotModifiedException(TableIdentifier tableIdentifier) {
+ super("Table " + tableIdentifier + " was not modified.");
+ }
+
+ public NotModifiedException(String message) {
+ super(message);
+ }
+
+ public NotModifiedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java
new file mode 100644
index 00000000..553d7253
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.Map;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.hadoop.Configurable;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.iceberg.rest.ResourcePaths;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.apache.iceberg.rest.responses.LoadTableResponseParser;
+import org.apache.polaris.tools.sync.polaris.http.OAuth2Util;
+
+/**
+ * Overrides loadTable default implementation to issue a custom loadTable request to the Polaris
+ * Iceberg REST Api and build the table metadata. This is necessary since the existing {@link
+ * RESTCatalog} does not provide a way to capture response headers to retrieve the ETag on a
+ * loadTable request.
+ */
+public class PolarisCatalog extends RESTCatalog
+ implements Catalog, ViewCatalog, SupportsNamespaces, Configurable