From 414f40e2dc5b46ed8750538ab8cd322044b4072d Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Sun, 20 Apr 2025 22:45:23 -0700 Subject: [PATCH 1/3] Added append only and remove strategies and put modification aware behind configurable flag --- .../sync/polaris/PolarisSynchronizer.java | 37 ++- ...nPlanner.java => BaseStrategyPlanner.java} | 96 ++++-- .../planning/SynchronizationPlanner.java | 4 +- ...ourceParitySynchronizationPlannerTest.java | 303 ------------------ .../AbstractBaseStrategyPlannerTest.java | 238 ++++++++++++++ ...teAndOverwriteBaseStrategyPlannerTest.java | 30 ++ .../CreateOnlyBaseStrategyPlannerTest.java | 30 ++ .../ReplicateBaseStrategyPlannerTest.java | 30 ++ .../sync/polaris/SyncPolarisCommand.java | 29 +- 9 files changed, 454 insertions(+), 343 deletions(-) rename polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/{SourceParitySynchronizationPlanner.java => BaseStrategyPlanner.java} (67%) delete mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateAndOverwriteBaseStrategyPlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateOnlyBaseStrategyPlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/ReplicateBaseStrategyPlannerTest.java diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index a521ecf3..8ebff7d5 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -61,13 +61,16 @@ public class PolarisSynchronizer { private final boolean haltOnFailure; + private final boolean deltaOnly; + public PolarisSynchronizer( Logger clientLogger, boolean haltOnFailure, SynchronizationPlanner synchronizationPlanner, PolarisService source, PolarisService target, - ETagManager etagManager) { + ETagManager etagManager, + boolean deltaOnly) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.haltOnFailure = haltOnFailure; @@ -75,6 +78,7 @@ public PolarisSynchronizer( this.source = source; this.target = target; this.etagManager = etagManager; + this.deltaOnly = deltaOnly; } /** @@ -1035,18 +1039,25 @@ public void syncNamespaces( try { Map sourceNamespaceMetadata = sourceIcebergCatalogService.loadNamespaceMetadata(namespace); - Map targetNamespaceMetadata = - targetIcebergCatalogService.loadNamespaceMetadata(namespace); - if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) { - clientLogger.info( - "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete); - continue; + if (this.deltaOnly) { + // if only configured to migrate the delta between the source and the target Polaris, + // then we can load the target namespace metadata and perform a comparison to discontinue early + // if we notice the metadata did not change + + Map targetNamespaceMetadata = + targetIcebergCatalogService.loadNamespaceMetadata(namespace); + + if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) { + clientLogger.info( + "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + continue; + } } targetIcebergCatalogService.setNamespaceProperties(namespace, sourceNamespaceMetadata); @@ -1206,7 +1217,7 @@ public void syncTables( try { Table table; - if (sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) { + if (this.deltaOnly && sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) { String etag = etagManager.getETag(catalogName, tableId); table = polarisCatalogService.loadTable(tableId, etag); } else { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/BaseStrategyPlanner.java similarity index 67% rename from polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java rename to polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/BaseStrategyPlanner.java index a60fea7d..39a2bac4 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/BaseStrategyPlanner.java @@ -33,21 +33,50 @@ import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; /** - * Sync planner that attempts to create total parity between the source and target Polaris - * instances. This involves creating new entities, overwriting entities that exist on both source - * and target, and removing entities that exist only on the target. + * Planner that implements the base level strategy that can be applied to synchronize the source and target. + * Can be configured at different levels of modification. */ -public class SourceParitySynchronizationPlanner implements SynchronizationPlanner { +public class BaseStrategyPlanner implements SynchronizationPlanner { /** - * Sort entities from the source into create, overwrite, and remove categories + * The strategy to employ when using {@link BaseStrategyPlanner}. + */ + public enum Strategy { + + /** + * Only create entities that exist on source but don't already exist on the target + */ + CREATE_ONLY, + + /** + * Create entities that do not exist on the target, and overwrite existing ones with same name/identifier + */ + CREATE_AND_OVERWRITE, + + /** + * Create entities that exist on the source and not target, update entities that exist on both, remove entities + * from the target that do not exist on the source. + */ + REPLICATE + + } + + private final Strategy strategy; + + public BaseStrategyPlanner(Strategy strategy) { + this.strategy = strategy; + } + + /** + * Sort entities from the source into create, overwrite, remove, and skip categories * on the basis of which identifiers exist on the source and target Polaris. * Identifiers that are both on the source and target instance will be marked - * for overwrite. Entities that are only on the source instance will be marked for - * creation. Entities that are only on the target instance will be marked for deletion. + * for overwrite if overwriting is enabled. Entities that are only on the source instance + * will be marked for creation. Entities that are only on the target instance will be marked for deletion + * only if the {@link Strategy#REPLICATE} strategy is used. * @param entitiesOnSource the entities from the source * @param entitiesOnTarget the entities from the target - * @param supportOverwrites true if "overwriting" the entity is necessary. Most grant record entities do not need overwriting. + * @param requiresOverwrites true if "overwriting" the entity is necessary. Most grant record entities do not need overwriting. * @param entityIdentifierSupplier consumes an entity and returns an identifying representation of that entity * @return a {@link SynchronizationPlan} with the entities sorted based on the souce parity strategy * @param the type of the entity @@ -55,7 +84,7 @@ public class SourceParitySynchronizationPlanner implements SynchronizationPlanne private SynchronizationPlan sortOnIdentifier( Collection entitiesOnSource, Collection entitiesOnTarget, - boolean supportOverwrites, + boolean requiresOverwrites, Function entityIdentifierSupplier ) { Set sourceEntityIdentifiers = entitiesOnSource.stream().map(entityIdentifierSupplier).collect(Collectors.toSet()); @@ -65,11 +94,28 @@ private SynchronizationPlan sortOnIdentifier( for (T entityOnSource : entitiesOnSource) { Object sourceEntityId = entityIdentifierSupplier.apply(entityOnSource); + if (targetEntityIdentifiers.contains(sourceEntityId)) { - if (supportOverwrites) { + // If an entity with this identifier exists on both the source and the target + + if (strategy == Strategy.CREATE_ONLY) { + // if the same entity identifier is on the source and target, + // but we only permit creates, skip it + plan.skipEntity(entityOnSource); + } else { // if the same entity identifier is on the source and the target, // overwrite the one on the target with the one on the source - plan.overwriteEntity(entityOnSource); + + if (requiresOverwrites) { + // If the entity requires a drop-and-recreate to perform an overwrite. + // some grant records can be "created" indefinitely even if they already exists, for example, + // catalog roles can be assigned the same principal role many times + plan.overwriteEntity(entityOnSource); + } else { + // if the entity is not a type that requires "overwriting" in the sense of + // dropping and recreating, then just create it again + plan.createEntity(entityOnSource); + } } } else { // if the entity identifier only exists on the source, that means @@ -89,7 +135,15 @@ private SynchronizationPlan sortOnIdentifier( // or a catalog role was revoked from a principal role, in which case the target // should reflect this change when the tool is run multiple times, because we don't // want to take chances with over-extending privileges - plan.removeEntity(entityOnTarget); + + if (strategy == Strategy.REPLICATE) { + plan.removeEntity(entityOnTarget); + } else { + // skip children here because if we want to remove the entity + // and then that means it does not exist on the source, so there are no child + // entities to sync + plan.skipEntityAndSkipChildren(entityOnTarget); + } } } @@ -99,7 +153,7 @@ private SynchronizationPlan sortOnIdentifier( @Override public SynchronizationPlan planPrincipalSync( List principalsOnSource, List principalsOnTarget) { - return sortOnIdentifier(principalsOnSource, principalsOnTarget, /* supportsOverwrites */ true, Principal::getName); + return sortOnIdentifier(principalsOnSource, principalsOnTarget, /* requiresOverwrites */ true, Principal::getName); } @Override @@ -111,7 +165,7 @@ public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSy return sortOnIdentifier( assignedPrincipalRolesOnSource, assignedPrincipalRolesOnTarget, - /* supportsOverwrites */ false, // do not need to overwrite an assignment of a principal role to a principal + /* requiresOverwrites */ false, // do not need to overwrite an assignment of a principal role to a principal PrincipalRole::getName ); } @@ -123,7 +177,7 @@ public SynchronizationPlan planPrincipalRoleSync( return sortOnIdentifier( principalRolesOnSource, principalRolesOnTarget, - /* supportsOverwrites */ true, + /* requiresOverwrites */ true, PrincipalRole::getName ); } @@ -131,7 +185,7 @@ public SynchronizationPlan planPrincipalRoleSync( @Override public SynchronizationPlan planCatalogSync( List catalogsOnSource, List catalogsOnTarget) { - return sortOnIdentifier(catalogsOnSource, catalogsOnTarget, /* supportsOverwrites */ true, Catalog::getName); + return sortOnIdentifier(catalogsOnSource, catalogsOnTarget, /* requiresOverwrites */ true, Catalog::getName); } @Override @@ -140,7 +194,7 @@ public SynchronizationPlan planCatalogRoleSync( List catalogRolesOnSource, List catalogRolesOnTarget) { return sortOnIdentifier( - catalogRolesOnSource, catalogRolesOnTarget, /* supportsOverwrites */ true, CatalogRole::getName); + catalogRolesOnSource, catalogRolesOnTarget, /* requiresOverwrites */ true, CatalogRole::getName); } @Override @@ -152,7 +206,7 @@ public SynchronizationPlan planGrantSync( return sortOnIdentifier( grantsOnSource, grantsOnTarget, - /* supportsOverwrites */ false, + /* requiresOverwrites */ false, grant -> grant // grants can just be compared by the entire generated object ); } @@ -166,7 +220,7 @@ public SynchronizationPlan planAssignPrincipalRolesToCatalogRoles return sortOnIdentifier( assignedPrincipalRolesOnSource, assignedPrincipalRolesOnTarget, - /* supportsOverwrites */ false, + /* requiresOverwrites */ false, PrincipalRole::getName ); } @@ -177,7 +231,7 @@ public SynchronizationPlan planNamespaceSync( Namespace namespace, List namespacesOnSource, List namespacesOnTarget) { - return sortOnIdentifier(namespacesOnSource, namespacesOnTarget, /* supportsOverwrites */ true, ns -> ns); + return sortOnIdentifier(namespacesOnSource, namespacesOnTarget, /* requiresOverwrites */ true, ns -> ns); } @Override @@ -187,6 +241,6 @@ public SynchronizationPlan planTableSync( Set tablesOnSource, Set tablesOnTarget) { return sortOnIdentifier( - tablesOnSource, tablesOnTarget, /* supportsOverwrites */ true, tableId -> tableId); + tablesOnSource, tablesOnTarget, /* requiresOverwrites */ true, tableId -> tableId); } } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java index 842cdcfa..8f769456 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java @@ -53,7 +53,7 @@ public interface PlannerWrapper { private final List plannerWrappers = new ArrayList<>(); - private SynchronizationPlannerBuilder(SourceParitySynchronizationPlanner innermost) { + private SynchronizationPlannerBuilder(BaseStrategyPlanner innermost) { this.innermost = innermost; } @@ -90,7 +90,7 @@ public SynchronizationPlanner build() { } } - static SynchronizationPlannerBuilder builder(SourceParitySynchronizationPlanner innermost) { + static SynchronizationPlannerBuilder builder(BaseStrategyPlanner innermost) { return new SynchronizationPlannerBuilder(innermost); } diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java deleted file mode 100644 index 18a961e7..00000000 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris; - -import java.util.List; -import java.util.Set; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; -import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class SourceParitySynchronizationPlannerTest { - - private static final Catalog CATALOG_1 = new Catalog().name("catalog-1"); - - private static final Catalog CATALOG_2 = new Catalog().name("catalog-2"); - - private static final Catalog CATALOG_3 = new Catalog().name("catalog-3"); - - @Test - public void testCreatesNewCatalogOverwritesOldCatalogRemovesDroppedCatalog() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planCatalogSync(List.of(CATALOG_1, CATALOG_2), List.of(CATALOG_2, CATALOG_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_3)); - } - - private static final Principal PRINCIPAL_1 = - new Principal().name("principal-1"); - - private static final Principal PRINCIPAL_2 = - new Principal().name("principal-2"); - - private static final Principal PRINCIPAL_3 = - new Principal().name("principal-3"); - - @Test - public void testCreatesNewPrincipalOverwritesOldPrincipalRemovesDroppedPrincipal() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planPrincipalSync(List.of(PRINCIPAL_1, PRINCIPAL_2), List.of(PRINCIPAL_2, PRINCIPAL_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_3)); - } - - private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_1 = - new PrincipalRole().name("principal-role-1"); - - private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_2 = - new PrincipalRole().name("principal-role-2"); - - private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_3 = - new PrincipalRole().name("principal-role-3"); - - @Test - public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRoleForPrincipal() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planAssignPrincipalsToPrincipalRolesSync( - "principal", - List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2), - List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1)); - - // special case: no concept of overwriting the assignment of a principal role - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3)); - } - - private static final PrincipalRole PRINCIPAL_ROLE_1 = - new PrincipalRole().name("principal-role-1"); - - private static final PrincipalRole PRINCIPAL_ROLE_2 = - new PrincipalRole().name("principal-role-2"); - - private static final PrincipalRole PRINCIPAL_ROLE_3 = - new PrincipalRole().name("principal-role-3"); - - @Test - public void testCreatesNewPrincipalRoleOverwritesOldPrincipalRoleRemovesDroppedPrincipalRole() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planPrincipalRoleSync( - List.of(PRINCIPAL_ROLE_1, PRINCIPAL_ROLE_2), - List.of(PRINCIPAL_ROLE_2, PRINCIPAL_ROLE_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_3)); - } - - private static final CatalogRole CATALOG_ROLE_1 = new CatalogRole().name("catalog-role-1"); - - private static final CatalogRole CATALOG_ROLE_2 = new CatalogRole().name("catalog-role-2"); - - private static final CatalogRole CATALOG_ROLE_3 = new CatalogRole().name("catalog-role-3"); - - @Test - public void testCreatesNewCatalogRoleOverwritesOldCatalogRoleRemovesDroppedCatalogRole() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planCatalogRoleSync( - "catalog", - List.of(CATALOG_ROLE_1, CATALOG_ROLE_2), - List.of(CATALOG_ROLE_2, CATALOG_ROLE_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_ROLE_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_ROLE_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_ROLE_3)); - } - - private static final GrantResource GRANT_1 = - new GrantResource().type(GrantResource.TypeEnum.CATALOG); - - private static final GrantResource GRANT_2 = - new GrantResource().type(GrantResource.TypeEnum.NAMESPACE); - - private static final GrantResource GRANT_3 = - new GrantResource().type(GrantResource.TypeEnum.TABLE); - - @Test - public void testCreatesNewGrantResourceRemovesDroppedGrantResource() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planGrantSync( - "catalog", "catalogRole", List.of(GRANT_1, GRANT_2), List.of(GRANT_2, GRANT_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(GRANT_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_1)); - - // special case: no concept of overwriting a grant - Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_2)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(GRANT_3)); - } - - private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_1 = - new PrincipalRole().name("principal-role-1"); - - private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_2 = - new PrincipalRole().name("principal-role-2"); - - private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_3 = - new PrincipalRole().name("principal-role-3"); - - @Test - public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRole() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planAssignPrincipalRolesToCatalogRolesSync( - "catalog", - "catalogRole", - List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2), - List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1)); - - // special case: no concept of overwriting the assignment of a principal role - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3)); - } - - private static final Namespace NS_1 = Namespace.of("ns1"); - - private static final Namespace NS_2 = Namespace.of("ns2"); - - private static final Namespace NS_3 = Namespace.of("ns3"); - - @Test - public void testCreatesNewNamespaceOverwritesOldNamespaceDropsDroppedNamespace() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - SynchronizationPlan plan = - planner.planNamespaceSync( - "catalog", Namespace.empty(), List.of(NS_1, NS_2), List.of(NS_2, NS_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(NS_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(NS_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(NS_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(NS_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(NS_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(NS_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(NS_3)); - } - - private static final TableIdentifier TABLE_1 = TableIdentifier.of("ns", "table1"); - - private static final TableIdentifier TABLE_2 = TableIdentifier.of("ns", "table2"); - - private static final TableIdentifier TABLE_3 = TableIdentifier.of("ns", "table3"); - - @Test - public void - testCreatesNewTableIdentifierOverwritesOldTableIdentifierRevokesDroppedTableIdentifier() { - SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); - - SynchronizationPlan plan = - planner.planTableSync( - "catalog", Namespace.empty(), Set.of(TABLE_1, TABLE_2), Set.of(TABLE_2, TABLE_3)); - - Assertions.assertTrue(plan.entitiesToCreate().contains(TABLE_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_1)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_2)); - Assertions.assertTrue(plan.entitiesToOverwrite().contains(TABLE_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_2)); - - Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(TABLE_3)); - } -} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java new file mode 100644 index 00000000..00343518 --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.tools.sync.polaris.strategy; + +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.BaseStrategyPlanner; +import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +public abstract class AbstractBaseStrategyPlannerTest { + + private final BaseStrategyPlanner.Strategy strategy; + + protected AbstractBaseStrategyPlannerTest(BaseStrategyPlanner.Strategy strategy) { + this.strategy = strategy; + } + + protected final Catalog CATALOG_1 = new Catalog().name("catalog-1"); + + protected final Catalog CATALOG_2 = new Catalog().name("catalog-2"); + + protected final Catalog CATALOG_3 = new Catalog().name("catalog-3"); + + protected void testStrategy(Function> planSupplier, + Object entityToCreate, + Object entityToOverwrite, + Object entityToRemove) { + testStrategy(planSupplier, true, entityToCreate, entityToOverwrite, entityToRemove); + } + + /** + * Test a generated plan for correctness in the case that there is 1 entity only on the source, + * 1 entity on both source and target, and 1 entity only on target. + * @param planSupplier generates the plan + * @param requiresOverwrite if the entity requires a drop and recreate (most grant records do not) + * @param entityOnSource the entity that is only on the source instance + * @param entityOnBoth the entity that is on both instances + * @param entityOnTarget the entity that is only on the target instance + */ + protected void testStrategy( + Function> planSupplier, + boolean requiresOverwrite, + Object entityOnSource, + Object entityOnBoth, + Object entityOnTarget + ) { + BaseStrategyPlanner planner = new BaseStrategyPlanner(strategy); + + SynchronizationPlan plan = planSupplier.apply(planner); + + Assertions.assertTrue(plan.entitiesToCreate().contains(entityOnSource)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(entityOnSource)); + Assertions.assertFalse(plan.entitiesToRemove().contains(entityOnSource)); + + if (!requiresOverwrite && strategy != BaseStrategyPlanner.Strategy.CREATE_ONLY) { + // if the entity is not one that needs to be overwritten, then any strategy + // besides CREATE_ONLY should instead schedule a "create" operation + Assertions.assertTrue(plan.entitiesToCreate().contains(entityOnBoth)); + } else { + Assertions.assertFalse(plan.entitiesToCreate().contains(entityOnBoth)); + } + + if (strategy == BaseStrategyPlanner.Strategy.CREATE_ONLY) { + // make sure entities to overwrite are skipped in CREATE_ONLY mode + Assertions.assertTrue(plan.entitiesToSkip().contains(entityOnBoth)); + } else if (requiresOverwrite) { + Assertions.assertTrue(plan.entitiesToOverwrite().contains(entityOnBoth)); + } + Assertions.assertFalse(plan.entitiesToRemove().contains(entityOnBoth)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(entityOnTarget)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(entityOnTarget)); + if (strategy == BaseStrategyPlanner.Strategy.REPLICATE) { + Assertions.assertTrue(plan.entitiesToRemove().contains(entityOnTarget)); + } else { + // only REPLICATE should remove entities from the target + Assertions.assertTrue(plan.entitiesToSkip().contains(entityOnTarget)); + } + } + + @Test + public void testCatalogStrategy() { + testStrategy(planner -> planner.planCatalogSync( + List.of(CATALOG_1, CATALOG_2), List.of(CATALOG_2, CATALOG_3)), + CATALOG_1, CATALOG_2, CATALOG_3); + } + + protected final Principal PRINCIPAL_1 = new Principal().name("principal-1"); + + protected final Principal PRINCIPAL_2 = new Principal().name("principal-2"); + + protected final Principal PRINCIPAL_3 = new Principal().name("principal-3"); + + @Test + public void testPrincipalStrategy() { + testStrategy(planner -> planner.planPrincipalSync( + List.of(PRINCIPAL_1, PRINCIPAL_2), List.of(PRINCIPAL_2, PRINCIPAL_3)), + PRINCIPAL_1, PRINCIPAL_2, PRINCIPAL_3); + } + + protected final PrincipalRole ASSIGNED_TO_PRINCIPAL_1 = new PrincipalRole().name("principal-role-1"); + + protected final PrincipalRole ASSIGNED_TO_PRINCIPAL_2 = new PrincipalRole().name("principal-role-2"); + + protected final PrincipalRole ASSIGNED_TO_PRINCIPAL_3 = new PrincipalRole().name("principal-role-3"); + + @Test + public void testAssignmentOfPrincipalRoleToPrincipalStrategy() { + testStrategy(planner -> + planner.planAssignPrincipalsToPrincipalRolesSync( + "principal", + List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2), + List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3)), + false, /* requiresOverwrite */ + ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3); + } + + protected final PrincipalRole PRINCIPAL_ROLE_1 = new PrincipalRole().name("principal-role-1"); + + protected final PrincipalRole PRINCIPAL_ROLE_2 = new PrincipalRole().name("principal-role-2"); + + protected final PrincipalRole PRINCIPAL_ROLE_3 = new PrincipalRole().name("principal-role-3"); + + @Test + public void testPrincipalRoleStrategy() { + testStrategy(planner -> planner.planPrincipalRoleSync( + List.of(PRINCIPAL_ROLE_1, PRINCIPAL_ROLE_2), + List.of(PRINCIPAL_ROLE_2, PRINCIPAL_ROLE_3)), + PRINCIPAL_ROLE_1, PRINCIPAL_ROLE_2, PRINCIPAL_ROLE_3); + } + + protected final CatalogRole CATALOG_ROLE_1 = new CatalogRole().name("catalog-role-1"); + + protected final CatalogRole CATALOG_ROLE_2 = new CatalogRole().name("catalog-role-2"); + + protected final CatalogRole CATALOG_ROLE_3 = new CatalogRole().name("catalog-role-3"); + + @Test + public void testCatalogRoleStrategy() { + testStrategy(planner -> + planner.planCatalogRoleSync( + "catalog", + List.of(CATALOG_ROLE_1, CATALOG_ROLE_2), + List.of(CATALOG_ROLE_2, CATALOG_ROLE_3)), + CATALOG_ROLE_1, CATALOG_ROLE_2, CATALOG_ROLE_3); + + } + + protected final GrantResource GRANT_1 = new GrantResource().type(GrantResource.TypeEnum.CATALOG); + + protected final GrantResource GRANT_2 = new GrantResource().type(GrantResource.TypeEnum.NAMESPACE); + + protected final GrantResource GRANT_3 = new GrantResource().type(GrantResource.TypeEnum.TABLE); + + @Test + public void testCreatesNewGrantResourceRemovesDroppedGrantResource() { + testStrategy(planner -> planner.planGrantSync( + "catalog", "catalogRole", + List.of(GRANT_1, GRANT_2), List.of(GRANT_2, GRANT_3)), + false, /* requiresOverwrite */ + GRANT_1, GRANT_2, GRANT_3); + } + + protected final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_1 = new PrincipalRole().name("principal-role-1"); + + protected final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_2 = new PrincipalRole().name("principal-role-2"); + + protected final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_3 = new PrincipalRole().name("principal-role-3"); + + @Test + public void testAssignPrincipalRoleToCatalogRoleStrategy() { + testStrategy(planner -> + planner.planAssignPrincipalRolesToCatalogRolesSync( + "catalog", + "catalogRole", + List.of(ASSIGNED_TO_CATALOG_ROLE_1, ASSIGNED_TO_CATALOG_ROLE_2), + List.of(ASSIGNED_TO_CATALOG_ROLE_2, ASSIGNED_TO_CATALOG_ROLE_3)), + false, /* requiresOverwrite */ + ASSIGNED_TO_CATALOG_ROLE_1, ASSIGNED_TO_CATALOG_ROLE_2, ASSIGNED_TO_CATALOG_ROLE_3); + } + + protected final Namespace NS_1 = Namespace.of("ns1"); + + protected final Namespace NS_2 = Namespace.of("ns2"); + + protected final Namespace NS_3 = Namespace.of("ns3"); + + @Test + public void testNamespaceStrategy() { + testStrategy(planner -> planner.planNamespaceSync( + "catalog", Namespace.empty(), List.of(NS_1, NS_2), List.of(NS_2, NS_3)), + NS_1, NS_2, NS_3); + } + + protected final TableIdentifier TABLE_1 = TableIdentifier.of("ns", "table1"); + + protected final TableIdentifier TABLE_2 = TableIdentifier.of("ns", "table2"); + + protected final TableIdentifier TABLE_3 = TableIdentifier.of("ns", "table3"); + + @Test + public void testTableStrategy() { + testStrategy(planner -> + planner.planTableSync( + "catalog", Namespace.empty(), Set.of(TABLE_1, TABLE_2), Set.of(TABLE_2, TABLE_3)), + TABLE_1, TABLE_2, TABLE_3); + } + +} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateAndOverwriteBaseStrategyPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateAndOverwriteBaseStrategyPlannerTest.java new file mode 100644 index 00000000..a2d41d59 --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateAndOverwriteBaseStrategyPlannerTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.tools.sync.polaris.strategy; + +import org.apache.polaris.tools.sync.polaris.planning.BaseStrategyPlanner; + +public class CreateAndOverwriteBaseStrategyPlannerTest extends AbstractBaseStrategyPlannerTest { + + protected CreateAndOverwriteBaseStrategyPlannerTest() { + super(BaseStrategyPlanner.Strategy.CREATE_AND_OVERWRITE); + } + +} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateOnlyBaseStrategyPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateOnlyBaseStrategyPlannerTest.java new file mode 100644 index 00000000..ebf16787 --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/CreateOnlyBaseStrategyPlannerTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.tools.sync.polaris.strategy; + +import org.apache.polaris.tools.sync.polaris.planning.BaseStrategyPlanner; + +public class CreateOnlyBaseStrategyPlannerTest extends AbstractBaseStrategyPlannerTest { + + protected CreateOnlyBaseStrategyPlannerTest() { + super(BaseStrategyPlanner.Strategy.CREATE_ONLY); + } + +} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/ReplicateBaseStrategyPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/ReplicateBaseStrategyPlannerTest.java new file mode 100644 index 00000000..11e83224 --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/ReplicateBaseStrategyPlannerTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.strategy; + + +import org.apache.polaris.tools.sync.polaris.planning.BaseStrategyPlanner; + +public class ReplicateBaseStrategyPlannerTest extends AbstractBaseStrategyPlannerTest { + + protected ReplicateBaseStrategyPlannerTest() { + super(BaseStrategyPlanner.Strategy.REPLICATE); + } + +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 31388ebe..a28b0cdb 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -24,7 +24,7 @@ import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; import org.apache.polaris.tools.sync.polaris.planning.CatalogNameFilterPlanner; import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; -import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.BaseStrategyPlanner; import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.service.PolarisService; import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService; @@ -98,10 +98,30 @@ public class SyncPolarisCommand implements Callable { ) private String catalogNameRegex; + @CommandLine.Option( + names = {"--delta-only"}, + description = "Only synchronize the diff between the source and target Polaris." + ) + private boolean deltaOnly; + + @CommandLine.Option( + names = {"--strategy"}, + defaultValue = "CREATE_ONLY", + description = "The synchronization strategy to use. Options: " + + "\n\t- CREATE_ONLY: (default) Only create entities that exist on the source and do not exist on the " + + "target." + + "\n\t- CREATE_AND_OVERWRITE: Create entities that exist on the source and not on the target and " + + "overwrite entities that exist on both the source and the target." + + "\n\t- REPLICATE: Create entities that exist on the source and not on the target, " + + "overwrite entities that exist on both the source and the target, " + + "and remove entities from the target that do not exist on the source." + ) + private BaseStrategyPlanner.Strategy strategy; + @Override public Integer call() throws Exception { - SynchronizationPlanner planner = SynchronizationPlanner.builder(new SourceParitySynchronizationPlanner()) - .wrapBy(ModificationAwarePlanner::new) + SynchronizationPlanner planner = SynchronizationPlanner.builder(new BaseStrategyPlanner(strategy)) + .conditionallyWrapBy(deltaOnly, ModificationAwarePlanner::new) .conditionallyWrapBy(catalogNameRegex != null, p -> new CatalogNameFilterPlanner(catalogNameRegex, p)) .wrapBy(AccessControlAwarePlanner::new) .build(); @@ -124,7 +144,8 @@ public Integer call() throws Exception { planner, source, target, - etagManager); + etagManager, + deltaOnly); synchronizer.syncPrincipalRoles(); if (shouldSyncPrincipals) { consoleLog.warn("Principal migration will reset credentials on the target Polaris instance. " + From 5a8d936d3a2b28a662b08b9d1b9f9d768efe163d Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 23 Apr 2025 18:34:52 -0700 Subject: [PATCH 2/3] Rename to diffOnly --- .../tools/sync/polaris/PolarisSynchronizer.java | 12 ++++++------ .../tools/sync/polaris/SyncPolarisCommand.java | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index 8ebff7d5..a7244370 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -61,7 +61,7 @@ public class PolarisSynchronizer { private final boolean haltOnFailure; - private final boolean deltaOnly; + private final boolean diffOnly; public PolarisSynchronizer( Logger clientLogger, @@ -70,7 +70,7 @@ public PolarisSynchronizer( PolarisService source, PolarisService target, ETagManager etagManager, - boolean deltaOnly) { + boolean diffOnly) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.haltOnFailure = haltOnFailure; @@ -78,7 +78,7 @@ public PolarisSynchronizer( this.source = source; this.target = target; this.etagManager = etagManager; - this.deltaOnly = deltaOnly; + this.diffOnly = diffOnly; } /** @@ -1040,8 +1040,8 @@ public void syncNamespaces( Map sourceNamespaceMetadata = sourceIcebergCatalogService.loadNamespaceMetadata(namespace); - if (this.deltaOnly) { - // if only configured to migrate the delta between the source and the target Polaris, + if (this.diffOnly) { + // if only configured to migrate the diff between the source and the target Polaris, // then we can load the target namespace metadata and perform a comparison to discontinue early // if we notice the metadata did not change @@ -1217,7 +1217,7 @@ public void syncTables( try { Table table; - if (this.deltaOnly && sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) { + if (this.diffOnly && sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) { String etag = etagManager.getETag(catalogName, tableId); table = polarisCatalogService.loadTable(tableId, etag); } else { diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index a28b0cdb..851d66f4 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -99,10 +99,10 @@ public class SyncPolarisCommand implements Callable { private String catalogNameRegex; @CommandLine.Option( - names = {"--delta-only"}, + names = {"--diff-only"}, description = "Only synchronize the diff between the source and target Polaris." ) - private boolean deltaOnly; + private boolean diffOnly; @CommandLine.Option( names = {"--strategy"}, @@ -121,7 +121,7 @@ public class SyncPolarisCommand implements Callable { @Override public Integer call() throws Exception { SynchronizationPlanner planner = SynchronizationPlanner.builder(new BaseStrategyPlanner(strategy)) - .conditionallyWrapBy(deltaOnly, ModificationAwarePlanner::new) + .conditionallyWrapBy(diffOnly, ModificationAwarePlanner::new) .conditionallyWrapBy(catalogNameRegex != null, p -> new CatalogNameFilterPlanner(catalogNameRegex, p)) .wrapBy(AccessControlAwarePlanner::new) .build(); @@ -145,7 +145,7 @@ public Integer call() throws Exception { source, target, etagManager, - deltaOnly); + diffOnly); synchronizer.syncPrincipalRoles(); if (shouldSyncPrincipals) { consoleLog.warn("Principal migration will reset credentials on the target Polaris instance. " + From 82c86fbe41302dd2ffaf5cffe416f06ad7d22a3a Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Fri, 25 Apr 2025 11:49:28 -0700 Subject: [PATCH 3/3] Changed test --- .../sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java index 00343518..e5b96795 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/strategy/AbstractBaseStrategyPlannerTest.java @@ -103,7 +103,7 @@ protected void testStrategy( Assertions.assertTrue(plan.entitiesToRemove().contains(entityOnTarget)); } else { // only REPLICATE should remove entities from the target - Assertions.assertTrue(plan.entitiesToSkip().contains(entityOnTarget)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(entityOnTarget)); } }