From f2cd4309e9559a8db7c542e90d0d4312e7b9bc36 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Tue, 8 Apr 2025 09:45:39 -0700 Subject: [PATCH 01/18] Initial commit of synchronizer code Addressed comments Add minimal build configuration --- polaris-synchronizer/.gitignore | 74 + polaris-synchronizer/README.md | 162 ++ polaris-synchronizer/api/build.gradle.kts | 154 ++ .../tools/sync/polaris/PolarisService.java | 371 +++++ .../sync/polaris/PolarisServiceFactory.java | 91 ++ .../sync/polaris/PolarisSynchronizer.java | 1135 +++++++++++++ .../access/AccessControlConstants.java | 26 + .../polaris/access/AccessControlService.java | 291 ++++ .../polaris/catalog/BaseTableWithETag.java | 44 + .../sync/polaris/catalog/ETagService.java | 46 + .../MetadataWrapperTableOperations.java | 68 + .../sync/polaris/catalog/NoOpETagService.java | 33 + .../polaris/catalog/NotModifiedException.java | 36 + .../sync/polaris/catalog/PolarisCatalog.java | 170 ++ .../tools/sync/polaris/http/HttpUtil.java | 39 + .../tools/sync/polaris/http/OAuth2Util.java | 78 + .../planning/AccessControlAwarePlanner.java | 198 +++ .../polaris/planning/DelegatedPlanner.java | 103 ++ .../planning/ModificationAwarePlanner.java | 329 ++++ .../polaris/planning/NoOpSyncPlanner.java | 88 + .../SourceParitySynchronizationPlanner.java | 250 +++ .../planning/SynchronizationPlanner.java | 71 + .../polaris/planning/plan/PlannedAction.java | 52 + .../planning/plan/SynchronizationPlan.java | 111 ++ .../resources/polaris-management-service.yml | 1432 +++++++++++++++++ .../AccessControlAwarePlannerTest.java | 149 ++ .../polaris/ModificationAwarePlannerTest.java | 302 ++++ ...ourceParitySynchronizationPlannerTest.java | 240 +++ polaris-synchronizer/build.gradle.kts | 24 + polaris-synchronizer/cli/build.gradle.kts | 69 + .../CreateOmnipotentPrincipalCommand.java | 171 ++ .../tools/sync/polaris/CsvETagService.java | 119 ++ .../sync/polaris/PolarisSynchronizerCLI.java | 43 + .../sync/polaris/SyncPolarisCommand.java | 132 ++ .../BaseOmnipotentPrincipalOptions.java | 53 + .../polaris/options/BasePolarisOptions.java | 72 + .../sync/polaris/options/PolarisOptions.java | 90 ++ .../SourceOmniPotentPrincipalOptions.java | 51 + .../polaris/options/SourcePolarisOptions.java | 90 ++ .../options/TargetOmnipotentPrincipal.java | 51 + .../polaris/options/TargetPolarisOptions.java | 90 ++ .../cli/src/main/resources/logback.xml | 36 + .../gradle/wrapper/gradle-wrapper.properties | 7 + polaris-synchronizer/gradlew | 252 +++ polaris-synchronizer/gradlew.bat | 94 ++ polaris-synchronizer/settings.gradle.kts | 10 + 46 files changed, 7597 insertions(+) create mode 100644 polaris-synchronizer/.gitignore create mode 100644 polaris-synchronizer/README.md create mode 100644 polaris-synchronizer/api/build.gradle.kts create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java create mode 100644 polaris-synchronizer/api/src/main/resources/polaris-management-service.yml create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java create mode 100644 polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java create mode 100644 polaris-synchronizer/build.gradle.kts create mode 100644 polaris-synchronizer/cli/build.gradle.kts create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java create mode 100644 polaris-synchronizer/cli/src/main/resources/logback.xml create mode 100644 polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties create mode 100755 polaris-synchronizer/gradlew create mode 100644 polaris-synchronizer/gradlew.bat create mode 100644 polaris-synchronizer/settings.gradle.kts diff --git a/polaris-synchronizer/.gitignore b/polaris-synchronizer/.gitignore new file mode 100644 index 00000000..a893541d --- /dev/null +++ b/polaris-synchronizer/.gitignore @@ -0,0 +1,74 @@ + +### Java ### +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +#misc +target/ +dependency-reduced-pom.xml +*.patch +*.DS_Store +.DS_Store + +#intellij +*.iml +.idea +*.ipr +*.iws + +# vscode +.vscode + +# node +node_modules/ +ui/src/generated/ + +# Eclipse IDE +.classpath +.factorypath +.project +.settings +.checkstyle +out/ + +# gradle +.gradle/ +build/ +gradle/wrapper/gradle-wrapper.jar +version.txt + +# Python venv +venv/ + +# Maven flatten plugin +.flattened-pom.xml + +# Site +site/site + +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md new file mode 100644 index 00000000..81e54a37 --- /dev/null +++ b/polaris-synchronizer/README.md @@ -0,0 +1,162 @@ +# Objective + +To provide users of [Apache Polaris (Incubating)](https://github.com/apache/polaris) a tool to be able to easily and efficiently +migrate their entities from one Polaris instance to another. + +Polaris is a catalog of catalogs. It can become cumbersome to perform catalog-by-catalog migration of each and every catalog contained +within a Polaris instance. Additionally, while migrating catalog-by-catalog Iceberg entities is achievable using the +existing generic [iceberg-catalog-migrator](../iceberg-catalog-migrator/README.md), the existing tool will not migrate +Polaris specific entities, like principal-roles, catalog-roles, grants. + +## Use Cases +* **Migration:** A user may have an active Polaris deployment that they want to migrate to a managed cloud offering like + [Snowflake Open Catalog](https://www.snowflake.com/en/product/features/open-catalog/). +* **Preventing Vendor Lock-In:** A user may currently have a managed Polaris offering and want the freedom to switch providers or to host Polaris themselves. +* **Mirroring/Disaster Recovery:** Modern data solutions require employing redundancy to ensure no single point of + failure. The tool can be scheduled on a cron to run periodic incremental syncs. + +The tool currently supports migrating the following Polaris Management entities: +* Principal roles +* Catalogs +* Catalog Roles +* Assignment of Catalog Roles to Principal Roles +* Grants + + +> :warning: Polaris principals and their assignments to Principal roles are not supported for migration +> by this tool. Migrating client credentials stored in Polaris is not possible nor is it secure. Polaris +> principals must be manually migrated between Polaris instances. + +The tool currently supports migrating the following Iceberg entities: +* Namespaces +* Tables + +# Building the Tool from Source + +**Prerequisite:** Must have Java installed in your machine (Java 21 is recommended and the minimum Java version) to use this CLI tool. + +``` +gradlew build # build and run tests +gradlew assemble # build without running tests +``` + +The default build location for the built JAR will be `cli/build/libs/` + +# Migrating between Polaris Instances + +### Step 1: Create a principal with read-only access to catalog internals on the source Polaris instance. + +**This step only has to be completed once.** + +Polaris is built with a separation between access and metadata management permissions. The `service_admin` +may have permissions to create access related entities like principal roles, catalog roles, and grants, but may not necessarily +possess the ability to view Iceberg content of catalogs, like namespaces and tables. We need to create a super user principal +that has access to all entities on the source Polaris instance in order to migrate them. + +To do this, we can use the `create-omnipotent-principal` command to create a principal, principal role, +and a catalog role per catalog with the appropriate grants to read all entities on the source Polaris instance. + +**Example:** Create a **read-only** principal on the source Polaris instance, and replace it if it already exists, +with 10 concurrent catalog setup threads: +``` +java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \ +--polaris-client-id root \ +--polaris-client-secret \ +--polaris-base-url http://localhost:8181 \ +--polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ +--polaris-scope PRINCIPAL_ROLE:ALL \ +--replace \ # replace if it exists +--concurrency 10 # 10 concurrent catalog setup threads +``` + +Upon finishing execution, the tool will output the principal name and client credentials for this +principal. **Make sure to note these down as they will be necessary for the migration step.** + +**Example Output:** +``` +====================================================== +Omnipotent Principal Credentials: +name = omnipotent-principal-XXXXX +clientId = ff7s8f9asbX10 +clientSecret = +====================================================== +``` + +Additionally, at the end of execution the command will output a list of catalogs for which catalog setup failed. +**These catalogs may experience failure during migration**. + +**Example Output:** +``` +Encountered issues creating catalog roles for the following catalogs: [catalog-1, catalog-2] +``` + +### Step 2: Create a principal with read-write access to catalog internals on the target Polaris instance. + +**This step only has to be completed once.** + +The same `create-omnipotent-principal` command can also be used to now create a **read-write** principal on the target +Polaris instance so that the tool can create entities on the target. + +To create a read-write principal, we simply specify the `--write-access` option. + +**Example:** Create a read-write principal on your target Polaris instance, replacing it if it exists, with 10 concurrent +catalog setup threads. +``` +java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \ +--polaris-client-id root \ +--polaris-client-secret \ +--polaris-base-url http://localhost:8181 \ +--polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ +--polaris-scope PRINCIPAL_ROLE:ALL \ +--replace \ # replace if it exists +--concurrency 10 \ # 10 concurrent catalog setup threads +--write-access # give the principal write access to catalog internals +``` + +Similarly to the last step, the tool will output the client credentials and principal name. Again, these need to be noted +for subsequent steps. + +**Example Output:** +``` +====================================================== +Omnipotent Principal Credentials: +name = omnipotent-principal-YYYYY +clientId = 0af20a3a0037a40d +clientSecret = +====================================================== +``` + +> :warning: `service_admin` is not guaranteed to have access management level grants on every catalog. This is usually +> delegated to the `catalog_admin` role, which is automatically granted to whichever principal role was used to create +> the catalog. This means that while the tool can detect this catalog when run with `service_admin` level access, +> it cannot create an omnipotent principal for this catalog. To remedy this, create a catalog-role with `CATALOG_MANAGE_ACCESS` +> grants for the catalog, and assign it to the principal used to run this tool (presumably, a principal with the `servic_admin` +> principal role). Then, re-running `create-omnipotent-principal` should be able to create the relevant entities for that catalog. + +### Step 3: Running the Migration/Synchronization + +Running the synchronization requires minimal reconfiguration, can be run idempotently, and will attempt to only copy over the +diff between the source and target Polaris instances. This can be achieved using the `sync-polaris` command. + +**Example** Running the synchronization between source Polaris instance using an access token, and a target Polaris instance +using client credentials. +``` +java -jar polaris-synchronizer-cli.jar sync-polaris \ +--source-base-url http://localhost:8182 \ +--source-access-token \ +--target-base-url http://localhost:8181 \ +--target-client-id root \ +--target-client-secret \ +--target-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ +--target-scope PRINCIPAL_ROLE:ALL \ +--source-omni-principal-name omnipotent-principal-XXXXX \ +--source-omni-client-id ff7s8f9asbX10 \ +--source-omni-client-secret \ +--target-omni-principal-name omnipotent-principal-YYYYY \ +--target-omni-client-id 0af20a3a0037a40d \ +--target-omni-client-secret +``` + +> :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source +> nor remove or modify them on the target. This is to accommodate that the tool itself will be running with the permission +> levels for these principals and roles, and we do not want to modify the tool's permissions at runtime. diff --git a/polaris-synchronizer/api/build.gradle.kts b/polaris-synchronizer/api/build.gradle.kts new file mode 100644 index 00000000..a348d215 --- /dev/null +++ b/polaris-synchronizer/api/build.gradle.kts @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + `java-library` + `maven-publish` + signing + id("org.openapi.generator") version "7.11.0" +} + +repositories { + mavenCentral() +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) // Set the compilation JDK to 21 + } +} + +dependencies { + // implementation(libs.openapi.generator) + implementation("jakarta.annotation:jakarta.annotation-api:2.1.1") + implementation("org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.7.1") + implementation("com.fasterxml.jackson.core:jackson-databind:2.18.3") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.3") + implementation("org.slf4j:log4j-over-slf4j:2.0.17") + + implementation("org.apache.hadoop:hadoop-common:2.7.3") { + exclude("org.apache.avro", "avro") + exclude("org.slf4j", "slf4j-log4j12") + exclude("javax.servlet", "servlet-api") + exclude("com.google.code.gson", "gson") + exclude("commons-beanutils") + } + + testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.0") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0") + testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.10.0") +} + +tasks.withType().configureEach { useJUnitPlatform() } + +tasks.register( + "generatePolarisManagementClient" +) { + inputSpec.set("$projectDir/src/main/resources/polaris-management-service.yml") + generatorName.set("java") + outputDir.set("${layout.buildDirectory.get()}/generated") + apiPackage.set("org.apache.polaris.management.client") + modelPackage.set("org.apache.polaris.core.admin.model") + removeOperationIdPrefix.set(true) + + globalProperties.set( + mapOf( + "apis" to "", + "models" to "", + "supportingFiles" to "", + "apiDocs" to "false", + "modelTests" to "false", + ) + ) + + additionalProperties.set( + mapOf( + "apiNamePrefix" to "PolarisManagement", + "apiNameSuffix" to "Api", + "metricsPrefix" to "polaris.management", + ) + ) + + configOptions.set( + mapOf( + "library" to "native", + "sourceFolder" to "src/main/java", + "useJakartaEe" to "true", + "useBeanValidation" to "false", + "openApiNullable" to "false", + "useRuntimeException" to "true", + "supportUrlQuery" to "false", + ) + ) + + importMappings.set( + mapOf( + "AbstractOpenApiSchema" to "org.apache.polaris.core.admin.model.AbstractOpenApiSchema", + "AddGrantRequest" to "org.apache.polaris.core.admin.model.AddGrantRequest", + "AwsStorageConfigInfo" to "org.apache.polaris.core.admin.model.AwsStorageConfigInfo", + "AzureStorageConfigInfo" to "org.apache.polaris.core.admin.model.AzureStorageConfigInfo", + "Catalog" to "org.apache.polaris.core.admin.model.Catalog", + "CatalogGrant" to "org.apache.polaris.core.admin.model.CatalogGrant", + "CatalogPrivilege" to "org.apache.polaris.core.admin.model.CatalogPrivilege", + "CatalogProperties" to "org.apache.polaris.core.admin.model.CatalogProperties", + "CatalogRole" to "org.apache.polaris.core.admin.model.CatalogRole", + "CatalogRoles" to "org.apache.polaris.core.admin.model.CatalogRoles", + "Catalogs" to "org.apache.polaris.core.admin.model.Catalogs", + "CreateCatalogRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRequest", + "CreateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRoleRequest", + "CreatePrincipalRequest" to "org.apache.polaris.core.admin.model.CreatePrincipalRequest", + "CreatePrincipalRoleRequest" to + "org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest", + "ExternalCatalog" to "org.apache.polaris.core.admin.model.ExternalCatalog", + "FileStorageConfigInfo" to "org.apache.polaris.core.admin.model.FileStorageConfigInfo", + "GcpStorageConfigInfo" to "org.apache.polaris.core.admin.model.GcpStorageConfigInfo", + "GrantCatalogRoleRequest" to "org.apache.polaris.core.admin.model.GrantCatalogRoleRequest", + "GrantPrincipalRoleRequest" to + "org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest", + "GrantResource" to "org.apache.polaris.core.admin.model.GrantResource", + "GrantResources" to "org.apache.polaris.core.admin.model.GrantResources", + "NamespaceGrant" to "org.apache.polaris.core.admin.model.NamespaceGrant", + "NamespacePrivilege" to "org.apache.polaris.core.admin.model.NamespacePrivilege", + "PolarisCatalog" to "org.apache.polaris.core.admin.model.PolarisCatalog", + "Principal" to "org.apache.polaris.core.admin.model.Principal", + "PrincipalRole" to "org.apache.polaris.core.admin.model.PrincipalRole", + "PrincipalRoles" to "org.apache.polaris.core.admin.model.PrincipalRoles", + "PrincipalWithCredentials" to "org.apache.polaris.core.admin.model.PrincipalWithCredentials", + "PrincipalWithCredentialsCredentials" to + "org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials", + "Principals" to "org.apache.polaris.core.admin.model.Principals", + "RevokeGrantRequest" to "org.apache.polaris.core.admin.model.RevokeGrantRequest", + "StorageConfigInfo" to "org.apache.polaris.core.admin.model.StorageConfigInfo", + "TableGrant" to "org.apache.polaris.core.admin.model.TableGrant", + "TablePrivilege" to "org.apache.polaris.core.admin.model.TablePrivilege", + "UpdateCatalogRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRequest", + "UpdateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest", + "UpdatePrincipalRequest" to "org.apache.polaris.core.admin.model.UpdatePrincipalRequest", + "UpdatePrincipalRoleRequest" to + "org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest", + "ViewGrant" to "org.apache.polaris.core.admin.model.ViewGrant", + "ViewPrivilege" to "org.apache.polaris.core.admin.model.ViewPrivilege", + ) + ) +} + +tasks.named("compileJava") { dependsOn("generatePolarisManagementClient") } + +sourceSets.main { java.srcDir("${layout.buildDirectory.get()}/generated/src/main/java") } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java new file mode 100644 index 00000000..18ff23c3 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.http.HttpStatus; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.AddGrantRequest; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; +import org.apache.polaris.core.admin.model.CreatePrincipalRequest; +import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.RevokeGrantRequest; +import org.apache.polaris.management.ApiException; +import org.apache.polaris.management.client.PolarisManagementDefaultApi; +import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; + +/** + * Service class that wraps Polaris HTTP client and performs recursive operations like drops on + * overwrites. + */ +public class PolarisService { + + private final PolarisManagementDefaultApi api; + + private final Map catalogProperties; + + public PolarisService(PolarisManagementDefaultApi api, Map catalogProperties) { + this.api = api; + this.catalogProperties = catalogProperties; + } + + public List listPrincipals() { + return this.api.listPrincipals().getPrincipals(); + } + + public Principal getPrincipal(String principalName) { + return this.api.getPrincipal(principalName); + } + + public boolean principalExists(String principalName) { + try { + getPrincipal(principalName); + return true; + } catch (ApiException apiException) { + if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { + return false; + } + throw apiException; + } + } + + public PrincipalWithCredentials createPrincipal(Principal principal, boolean overwrite) { + if (overwrite) { + removePrincipal(principal.getName()); + } + + CreatePrincipalRequest request = new CreatePrincipalRequest().principal(principal); + return this.api.createPrincipal(request); + } + + public void removePrincipal(String principalName) { + this.api.deletePrincipal(principalName); + } + + public void assignPrincipalRole(String principalName, String principalRoleName) { + GrantPrincipalRoleRequest request = + new GrantPrincipalRoleRequest().principalRole(new PrincipalRole().name(principalRoleName)); + this.api.assignPrincipalRole(principalName, request); + } + + public void createPrincipalRole(PrincipalRole principalRole, boolean overwrite) { + if (overwrite) { + removePrincipalRole(principalRole.getName()); + } + CreatePrincipalRoleRequest request = + new CreatePrincipalRoleRequest().principalRole(principalRole); + this.api.createPrincipalRole(request); + } + + public List listPrincipalRolesAssignedForPrincipal(String principalName) { + return this.api.listPrincipalRolesAssigned(principalName).getRoles(); + } + + public List listPrincipalRoles() { + return this.api.listPrincipalRoles().getRoles(); + } + + public List listAssigneePrincipalRolesForCatalogRole( + String catalogName, String catalogRoleName) { + return this.api + .listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName) + .getRoles(); + } + + public void assignCatalogRoleToPrincipalRole( + String principalRoleName, String catalogName, String catalogRoleName) { + GrantCatalogRoleRequest request = + new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName)); + this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request); + } + + public void removeCatalogRoleFromPrincipalRole( + String principalRoleName, String catalogName, String catalogRoleName) { + this.api.revokeCatalogRoleFromPrincipalRole(principalRoleName, catalogName, catalogRoleName); + } + + public PrincipalRole getPrincipalRole(String principalRoleName) { + return this.api.getPrincipalRole(principalRoleName); + } + + public boolean principalRoleExists(String principalRoleName) { + try { + getPrincipalRole(principalRoleName); + return true; + } catch (ApiException apiException) { + if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { + return false; + } + throw apiException; + } + } + + public void removePrincipalRole(String principalRoleName) { + this.api.deletePrincipalRole(principalRoleName); + } + + public List listCatalogs() { + return this.api.listCatalogs().getCatalogs(); + } + + public void createCatalog(Catalog catalog) { + CreateCatalogRequest request = new CreateCatalogRequest().catalog(catalog); + this.api.createCatalog(request); + } + + /** + * Performs a cascading drop on the catalog before recreating. + * + * @param catalog + * @param omnipotentPrincipal necessary to initialize an Iceberg catalog to drop catalog internals + */ + public void overwriteCatalog(Catalog catalog, PrincipalWithCredentials omnipotentPrincipal) { + removeCatalogCascade(catalog.getName(), omnipotentPrincipal); + createCatalog(catalog); + } + + /** + * Recursively discover all namespaces contained within an Iceberg catalog. + * + * @param catalog + * @return a list of all the namespaces in the catalog + */ + private List discoverAllNamespaces(org.apache.iceberg.catalog.Catalog catalog) { + List namespaces = new ArrayList<>(); + namespaces.add(Namespace.empty()); + + if (catalog instanceof SupportsNamespaces namespaceCatalog) { + namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, Namespace.empty())); + } + + return namespaces; + } + + /** + * Discover all child namespaces of a given namespace. + * + * @param namespaceCatalog a catalog that supports nested namespaces + * @param namespace the namespace to look under + * @return a list of all child namespaces + */ + private List discoverContainedNamespaces( + SupportsNamespaces namespaceCatalog, Namespace namespace) { + List immediateChildren = namespaceCatalog.listNamespaces(namespace); + + List namespaces = new ArrayList<>(); + + for (Namespace ns : immediateChildren) { + namespaces.add(ns); + + // discover children of child namespace + namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, ns)); + } + + return namespaces; + } + + /** + * Perform a cascading drop of a catalog. Removes all namespaces, tables, catalog-roles first. + * + * @param catalogName + * @param omnipotentPrincipal + */ + public void removeCatalogCascade( + String catalogName, PrincipalWithCredentials omnipotentPrincipal) { + org.apache.iceberg.catalog.Catalog icebergCatalog = + initializeCatalog(catalogName, omnipotentPrincipal); + + // find all namespaces in the catalog + List namespaces = discoverAllNamespaces(icebergCatalog); + + List tables = new ArrayList<>(); + + // find all tables in the catalog + for (Namespace ns : namespaces) { + if (!ns.isEmpty()) { + tables.addAll(icebergCatalog.listTables(ns)); + } + } + + // drop every table in the catalog + for (TableIdentifier table : tables) { + icebergCatalog.dropTable(table); + } + + // drop every namespace in the catalog, note that because we discovered the namespaces + // parent-first, we should reverse over the namespaces to ensure that we drop child namespaces + // before we drop parent namespaces, as we cannot drop nonempty namespaces + for (Namespace ns : namespaces.reversed()) { + // NOTE: this is checking if the namespace is not the empty namespace, not if it is empty + // in the sense of containing no tables/namespaces + if (!ns.isEmpty() && icebergCatalog instanceof SupportsNamespaces namespaceCatalog) { + namespaceCatalog.dropNamespace(ns); + } + } + + List catalogRoles = listCatalogRoles(catalogName); + + // remove catalog roles under catalog + for (CatalogRole catalogRole : catalogRoles) { + if (catalogRole.getName().equals("catalog_admin")) continue; + + removeCatalogRole(catalogName, catalogRole.getName()); + } + + this.api.deleteCatalog(catalogName); + } + + public List listCatalogRoles(String catalogName) { + return this.api.listCatalogRoles(catalogName).getRoles(); + } + + public CatalogRole getCatalogRole(String catalogName, String catalogRoleName) { + return this.api.getCatalogRole(catalogName, catalogRoleName); + } + + public boolean catalogRoleExists(String catalogName, String catalogRoleName) { + try { + getCatalogRole(catalogName, catalogRoleName); + return true; + } catch (ApiException apiException) { + if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { + return false; + } + throw apiException; + } + } + + public void assignCatalogRole( + String principalRoleName, String catalogName, String catalogRoleName) { + GrantCatalogRoleRequest request = + new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName)); + this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request); + } + + public void createCatalogRole(String catalogName, CatalogRole catalogRole, boolean overwrite) { + if (overwrite) { + removeCatalogRole(catalogName, catalogRole.getName()); + } + + CreateCatalogRoleRequest request = new CreateCatalogRoleRequest().catalogRole(catalogRole); + this.api.createCatalogRole(catalogName, request); + } + + public void removeCatalogRole(String catalogName, String catalogRoleName) { + this.api.deleteCatalogRole(catalogName, catalogRoleName); + } + + public List listGrants(String catalogName, String catalogRoleName) { + return this.api.listGrantsForCatalogRole(catalogName, catalogRoleName).getGrants(); + } + + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + AddGrantRequest addGrantRequest = new AddGrantRequest().grant(grant); + this.api.addGrantToCatalogRole(catalogName, catalogRoleName, addGrantRequest); + } + + public void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant) { + RevokeGrantRequest revokeGrantRequest = new RevokeGrantRequest().grant(grant); + this.api.revokeGrantFromCatalogRole(catalogName, catalogRoleName, false, revokeGrantRequest); + } + + public org.apache.iceberg.catalog.Catalog initializeCatalog( + String catalogName, PrincipalWithCredentials migratorPrincipal) { + Map currentCatalogProperties = new HashMap<>(catalogProperties); + currentCatalogProperties.put("warehouse", catalogName); + + String clientId = migratorPrincipal.getCredentials().getClientId(); + String clientSecret = migratorPrincipal.getCredentials().getClientSecret(); + currentCatalogProperties.putIfAbsent( + "credential", String.format("%s:%s", clientId, clientSecret)); + currentCatalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL"); + + return CatalogUtil.loadCatalog( + PolarisCatalog.class.getName(), "SOURCE_CATALOG_REST", currentCatalogProperties, null); + } + + /** + * Perform cascading drop of a namespace. + * + * @param icebergCatalog the iceberg catalog to use + * @param namespace the namespace to drop + */ + public void dropNamespaceCascade( + org.apache.iceberg.catalog.Catalog icebergCatalog, Namespace namespace) { + if (icebergCatalog instanceof SupportsNamespaces namespaceCatalog) { + List namespaces = discoverContainedNamespaces(namespaceCatalog, namespace); + + List tables = new ArrayList<>(); + + for (Namespace ns : namespaces) { + tables.addAll(icebergCatalog.listTables(ns)); + } + + tables.addAll(icebergCatalog.listTables(namespace)); + + for (TableIdentifier table : tables) { + icebergCatalog.dropTable(table); + } + + // go over in reverse order of namespaces since we discover namespaces + // in the parent -> child order, so we need to drop all children + // before we can drop the parent + for (Namespace ns : namespaces.reversed()) { + namespaceCatalog.dropNamespace(ns); + } + + namespaceCatalog.dropNamespace(namespace); + } + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java new file mode 100644 index 00000000..72d0f78c --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.http.HttpHeaders; +import org.apache.polaris.management.ApiClient; +import org.apache.polaris.management.client.PolarisManagementDefaultApi; +import org.apache.polaris.tools.sync.polaris.http.OAuth2Util; + +/** Used to initialize a {@link PolarisService}. */ +public class PolarisServiceFactory { + + private static void validatePolarisInstanceProperties( + String baseUrl, + String accessToken, + String oauth2ServerUri, + String clientId, + String clientSecret, + String scope) { + if (baseUrl == null) { + throw new IllegalArgumentException("baseUrl is required but was not provided"); + } + + if (accessToken != null) { + return; + } + + final String oauthErrorMessage = + "Either the accessToken property must be provided, or all of oauth2ServerUri, clientId, clientSecret, scope"; + + if (oauth2ServerUri == null || clientId == null || clientSecret == null || scope == null) { + throw new IllegalArgumentException(oauthErrorMessage); + } + } + + public static PolarisService newPolarisService( + String baseUrl, String oauth2ServerUri, String clientId, String clientSecret, String scope) + throws IOException { + validatePolarisInstanceProperties( + baseUrl, null /* accessToken */, oauth2ServerUri, clientId, clientSecret, scope); + + String accessToken = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope); + + return newPolarisService(baseUrl, accessToken); + } + + public static PolarisService newPolarisService(String baseUrl, String accessToken) { + validatePolarisInstanceProperties( + baseUrl, + accessToken, + null, /* oauth2ServerUri */ + null, /* clientId */ + null, /* clientSecret */ + null /* scope */ + ); + + ApiClient client = new ApiClient(); + client.updateBaseUri(baseUrl + "/api/management/v1"); + + // TODO: Add token refresh + client.setRequestInterceptor( + requestBuilder -> { + requestBuilder.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); + }); + + Map catalogProperties = new HashMap<>(); + catalogProperties.putIfAbsent("uri", baseUrl + "/api/catalog"); + + PolarisManagementDefaultApi polarisClient = new PolarisManagementDefaultApi(client); + return new PolarisService(polarisClient, catalogProperties); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java new file mode 100644 index 00000000..b5954302 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -0,0 +1,1135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.access.AccessControlService; +import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag; +import org.apache.polaris.tools.sync.polaris.catalog.ETagService; +import org.apache.polaris.tools.sync.polaris.catalog.NotModifiedException; +import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; +import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Encapsulates idempotent and failure-safe logic to perform Polaris entity syncs. Performs logging + * with configurability and all actions related to the generated sync plans. + */ +public class PolarisSynchronizer { + + private final Logger clientLogger; + + private final SynchronizationPlanner syncPlanner; + + private final PolarisService source; + + private final PolarisService target; + + private final PrincipalWithCredentials sourceOmnipotentPrincipal; + + private final PrincipalWithCredentials targetOmnipotentPrincipal; + + private final PrincipalRole sourceOmnipotentPrincipalRole; + + private final PrincipalRole targetOmnipotentPrincipalRole; + + private final AccessControlService sourceAccessControlService; + + private final AccessControlService targetAccessControlService; + + private final ETagService etagService; + + public PolarisSynchronizer( + Logger clientLogger, + SynchronizationPlanner synchronizationPlanner, + PrincipalWithCredentials sourceOmnipotentPrincipal, + PrincipalWithCredentials targetOmnipotentPrincipal, + PolarisService source, + PolarisService target, + ETagService etagService) { + this.clientLogger = + clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; + this.syncPlanner = synchronizationPlanner; + this.sourceOmnipotentPrincipal = sourceOmnipotentPrincipal; + this.targetOmnipotentPrincipal = targetOmnipotentPrincipal; + this.source = source; + this.target = target; + this.sourceAccessControlService = new AccessControlService(source); + this.targetAccessControlService = new AccessControlService(target); + + this.sourceOmnipotentPrincipalRole = + sourceAccessControlService.getOmnipotentPrincipalRoleForPrincipal( + sourceOmnipotentPrincipal.getPrincipal().getName()); + this.targetOmnipotentPrincipalRole = + targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal( + targetOmnipotentPrincipal.getPrincipal().getName()); + this.etagService = etagService; + } + + /** + * Calculates the total number of sync tasks to complete. + * + * @param plan the plan to scan for cahnges + * @return the nuber of syncs to perform + */ + private int totalSyncsToComplete(SynchronizationPlan plan) { + return plan.entitiesToCreate().size() + + plan.entitiesToOverwrite().size() + + plan.entitiesToRemove().size(); + } + + /** Sync principal roles from source to target. */ + public void syncPrincipalRoles() { + List principalRolesSource; + + try { + principalRolesSource = source.listPrincipalRoles(); + clientLogger.info("Listed {} principal-roles from source.", principalRolesSource.size()); + } catch (Exception e) { + clientLogger.error("Failed to list principal-roles from source.", e); + return; + } + + List principalRolesTarget; + + try { + principalRolesTarget = target.listPrincipalRoles(); + clientLogger.info("Listed {} principal-roles from target.", principalRolesTarget.size()); + } catch (Exception e) { + clientLogger.error("Failed to list principal-roles from target.", e); + return; + } + + SynchronizationPlan principalRoleSyncPlan = + syncPlanner.planPrincipalRoleSync(principalRolesSource, principalRolesTarget); + + principalRoleSyncPlan + .entitiesToSkip() + .forEach( + principalRole -> + clientLogger.info("Skipping principal-role {}.", principalRole.getName())); + + principalRoleSyncPlan + .entitiesNotModified() + .forEach( + principalRole -> + clientLogger.info( + "No change detected for principal-role {}, skipping.", + principalRole.getName())); + + int syncsCompleted = 0; + final int totalSyncsToComplete = totalSyncsToComplete(principalRoleSyncPlan); + + for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToCreate()) { + try { + target.createPrincipalRole(principalRole, false); + clientLogger.info( + "Created principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to create principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToOverwrite()) { + try { + target.createPrincipalRole(principalRole, true); + clientLogger.info( + "Overwrote principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to overwrite principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToRemove()) { + try { + target.removePrincipalRole(principalRole.getName()); + clientLogger.info( + "Removed principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to remove principal-role {} on target. - {}/{}", + principalRole.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + } + + /** + * Sync assignments of principal roles to a catalog role. + * + * @param catalogName the catalog that the catalog role is in + * @param catalogRoleName the name of the catalog role + */ + public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName) { + List principalRolesSource; + + try { + principalRolesSource = + source.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName); + clientLogger.info( + "Listed {} assignee principal-roles for catalog-role {} in catalog {} from source.", + principalRolesSource.size(), + catalogRoleName, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list assignee principal-roles for catalog-role {} in catalog {} from source.", + catalogRoleName, + catalogName, + e); + return; + } + + List principalRolesTarget; + + try { + principalRolesTarget = + target.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName); + clientLogger.info( + "Listed {} assignee principal-roles for catalog-role {} in catalog {} from target.", + principalRolesTarget.size(), + catalogRoleName, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list assignee principal-roles for catalog-role {} in catalog {} from target.", + catalogRoleName, + catalogName, + e); + return; + } + + SynchronizationPlan assignedPrincipalRoleSyncPlan = + syncPlanner.planAssignPrincipalRolesToCatalogRolesSync( + catalogName, catalogRoleName, principalRolesSource, principalRolesTarget); + + assignedPrincipalRoleSyncPlan + .entitiesToSkip() + .forEach( + principalRole -> + clientLogger.info( + "Skipping assignment of principal-role {} to catalog-role {} in catalog {}.", + principalRole.getName(), + catalogRoleName, + catalogName)); + + assignedPrincipalRoleSyncPlan + .entitiesNotModified() + .forEach( + principalRole -> + clientLogger.info( + "Principal-role {} is already assigned to catalog-role {} in catalog {}. Skipping.", + principalRole.getName(), + catalogRoleName, + catalogName)); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(assignedPrincipalRoleSyncPlan); + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) { + try { + target.assignCatalogRoleToPrincipalRole( + principalRole.getName(), catalogName, catalogRoleName); + clientLogger.info( + "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) { + try { + target.assignCatalogRoleToPrincipalRole( + principalRole.getName(), catalogName, catalogRoleName); + clientLogger.info( + "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) { + try { + target.removeCatalogRoleFromPrincipalRole( + principalRole.getName(), catalogName, catalogRoleName); + clientLogger.info( + "Revoked principal-role {} from catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to revoke principal-role {} from catalog-role {} in catalog {}. - {}/{}", + principalRole.getName(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + } + + /** Sync catalogs across the source and target polaris instance. */ + public void syncCatalogs() { + List catalogsSource; + + try { + catalogsSource = source.listCatalogs(); + clientLogger.info("Listed {} catalogs from source.", catalogsSource.size()); + } catch (Exception e) { + clientLogger.error("Failed to list catalogs from source.", e); + return; + } + + List catalogsTarget; + + try { + catalogsTarget = target.listCatalogs(); + clientLogger.info("Listed {} catalogs from target.", catalogsTarget.size()); + } catch (Exception e) { + clientLogger.error("Failed to list catalogs from target.", e); + return; + } + + SynchronizationPlan catalogSyncPlan = + syncPlanner.planCatalogSync(catalogsSource, catalogsTarget); + + catalogSyncPlan + .entitiesToSkip() + .forEach(catalog -> clientLogger.info("Skipping catalog {}.", catalog.getName())); + + catalogSyncPlan + .entitiesToSkipAndSkipChildren() + .forEach( + catalog -> + clientLogger.info( + "Skipping catalog {} and all child entities.", catalog.getName())); + + catalogSyncPlan + .entitiesNotModified() + .forEach( + catalog -> + clientLogger.info( + "No change detected in catalog {}. Skipping.", catalog.getName())); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(catalogSyncPlan); + + for (Catalog catalog : catalogSyncPlan.entitiesToCreate()) { + try { + target.createCatalog(catalog); + clientLogger.info( + "Created catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to create catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Catalog catalog : catalogSyncPlan.entitiesToOverwrite()) { + try { + setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName()); + target.overwriteCatalog(catalog, targetOmnipotentPrincipal); + clientLogger.info( + "Overwrote catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to overwrite catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Catalog catalog : catalogSyncPlan.entitiesToRemove()) { + try { + setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName()); + target.removeCatalogCascade(catalog.getName(), targetOmnipotentPrincipal); + clientLogger.info( + "Removed catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to remove catalog {}. - {}/{}", + catalog.getName(), + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Catalog catalog : catalogSyncPlan.entitiesToSyncChildren()) { + syncCatalogRoles(catalog.getName()); + + org.apache.iceberg.catalog.Catalog sourceIcebergCatalog; + + try { + sourceIcebergCatalog = initializeIcebergCatalogSource(catalog.getName()); + clientLogger.info( + "Initialized Iceberg REST catalog for Polaris catalog {} on source.", + catalog.getName()); + } catch (Exception e) { + clientLogger.error( + "Failed to initialize Iceberg REST catalog for Polaris catalog {} on source.", + catalog.getName(), + e); + continue; + } + + org.apache.iceberg.catalog.Catalog targetIcebergCatalog; + + try { + targetIcebergCatalog = initializeIcebergCatalogTarget(catalog.getName()); + clientLogger.info( + "Initialized Iceberg REST catalog for Polaris catalog {} on target.", + catalog.getName()); + } catch (Exception e) { + clientLogger.error( + "Failed to initialize Iceberg REST catalog for Polaris catalog {} on target.", + catalog.getName(), + e); + continue; + } + + syncNamespaces( + catalog.getName(), Namespace.empty(), sourceIcebergCatalog, targetIcebergCatalog); + } + } + + /** + * Sync catalog roles across the source and polaris instance for a catalog. + * + * @param catalogName the catalog to sync roles for + */ + public void syncCatalogRoles(String catalogName) { + List catalogRolesSource; + + try { + catalogRolesSource = source.listCatalogRoles(catalogName); + clientLogger.info( + "Listed {} catalog-roles for catalog {} from source.", + catalogRolesSource.size(), + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list catalog-roles for catalog {} from source.", catalogName, e); + return; + } + + List catalogRolesTarget; + + try { + catalogRolesTarget = target.listCatalogRoles(catalogName); + clientLogger.info( + "Listed {} catalog-roles for catalog {} from target.", + catalogRolesTarget.size(), + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list catalog-roles for catalog {} from target.", catalogName, e); + return; + } + + SynchronizationPlan catalogRoleSyncPlan = + syncPlanner.planCatalogRoleSync(catalogName, catalogRolesSource, catalogRolesTarget); + + catalogRoleSyncPlan + .entitiesToSkip() + .forEach( + catalogRole -> + clientLogger.info( + "Skipping catalog-role {} in catalog {}.", catalogRole.getName(), catalogName)); + + catalogRoleSyncPlan + .entitiesToSkipAndSkipChildren() + .forEach( + catalogRole -> + clientLogger.info( + "Skipping catalog-role {} in catalog {} and all child entities.", + catalogRole.getName(), + catalogName)); + + catalogRoleSyncPlan + .entitiesNotModified() + .forEach( + catalogRole -> + clientLogger.info( + "No change detected in catalog-role {} in catalog {}. Skipping.", + catalogRole.getName(), + catalogName)); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(catalogRoleSyncPlan); + + for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToCreate()) { + try { + target.createCatalogRole(catalogName, catalogRole, false); + clientLogger.info( + "Created catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to create catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToOverwrite()) { + try { + target.createCatalogRole(catalogName, catalogRole, true); + clientLogger.info( + "Overwrote catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to overwrite catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToRemove()) { + try { + target.removeCatalogRole(catalogName, catalogRole.getName()); + clientLogger.info( + "Removed catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to remove catalog-role {} for catalog {}. - {}/{}", + catalogRole.getName(), + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToSyncChildren()) { + syncAssigneePrincipalRolesForCatalogRole(catalogName, catalogRole.getName()); + syncGrants(catalogName, catalogRole.getName()); + } + } + + /** + * Sync grants for a catalog role across the source and the target. + * + * @param catalogName + * @param catalogRoleName + */ + private void syncGrants(String catalogName, String catalogRoleName) { + List grantsSource; + + try { + grantsSource = source.listGrants(catalogName, catalogRoleName); + clientLogger.info( + "Listed {} grants for catalog-role {} in catalog {} from source.", + grantsSource.size(), + catalogRoleName, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list grants for catalog-role {} in catalog {} from source.", + catalogRoleName, + catalogName, + e); + return; + } + + List grantsTarget; + + try { + grantsTarget = target.listGrants(catalogName, catalogRoleName); + clientLogger.info( + "Listed {} grants for catalog-role {} in catalog {} from target.", + grantsTarget.size(), + catalogRoleName, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list grants for catalog-role {} in catalog {} from target.", + catalogRoleName, + catalogName, + e); + return; + } + + SynchronizationPlan grantSyncPlan = + syncPlanner.planGrantSync(catalogName, catalogRoleName, grantsSource, grantsTarget); + + grantSyncPlan + .entitiesToSkip() + .forEach( + grant -> + clientLogger.info( + "Skipping addition of grant {} to catalog-role {} in catalog {}.", + grant.getType(), + catalogRoleName, + catalogName)); + + grantSyncPlan + .entitiesNotModified() + .forEach( + grant -> + clientLogger.info( + "Grant {} was already added to catalog-role {} in catalog {}. Skipping.", + grant.getType(), + catalogRoleName, + catalogName)); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(grantSyncPlan); + + for (GrantResource grant : grantSyncPlan.entitiesToCreate()) { + try { + target.addGrant(catalogName, catalogRoleName, grant); + clientLogger.info( + "Added grant {} to catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (GrantResource grant : grantSyncPlan.entitiesToOverwrite()) { + try { + target.addGrant(catalogName, catalogRoleName, grant); + clientLogger.info( + "Added grant {} to catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (GrantResource grant : grantSyncPlan.entitiesToRemove()) { + try { + target.revokeGrant(catalogName, catalogRoleName, grant); + clientLogger.info( + "Revoked grant {} from catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to revoke grant {} from catalog-role {} for catalog {}. - {}/{}", + grant.getType(), + catalogRoleName, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + } + + /** + * Setup an omnipotent principal for the provided catalog on the target Polaris instance. + * + * @param catalogName + */ + private void setupOmnipotentCatalogRoleIfNotExistsTarget(String catalogName) { + if (!this.targetAccessControlService.omnipotentCatalogRoleExists(catalogName)) { + clientLogger.info( + "No omnipotent catalog-role exists for catalog {} on target. Going to set one up.", + catalogName); + + targetAccessControlService.setupOmnipotentRoleForCatalog( + catalogName, targetOmnipotentPrincipalRole, false, true); + + clientLogger.info("Setup omnipotent catalog-role for catalog {} on target.", catalogName); + } + } + + /** + * Setup an omnipotent principal for the provided catalog on the target Polaris instance. + * + * @param catalogName + */ + private void setupOmnipotentCatalogRoleIfNotExistsSource(String catalogName) { + if (!this.sourceAccessControlService.omnipotentCatalogRoleExists(catalogName)) { + clientLogger.info( + "No omnipotent catalog-role exists for catalog {} on source. Going to set one up.", + catalogName); + + sourceAccessControlService.setupOmnipotentRoleForCatalog( + catalogName, sourceOmnipotentPrincipalRole, false, false); + + clientLogger.info("Setup omnipotent catalog-role for catalog {} on source.", catalogName); + } + } + + public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogSource(String catalogName) { + setupOmnipotentCatalogRoleIfNotExistsSource(catalogName); + return source.initializeCatalog(catalogName, sourceOmnipotentPrincipal); + } + + public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogTarget(String catalogName) { + setupOmnipotentCatalogRoleIfNotExistsTarget(catalogName); + return target.initializeCatalog(catalogName, targetOmnipotentPrincipal); + } + + /** + * Sync namespaces contained within a parent namespace. + * + * @param catalogName + * @param parentNamespace + * @param sourceIcebergCatalog + * @param targetIcebergCatalog + */ + public void syncNamespaces( + String catalogName, + Namespace parentNamespace, + org.apache.iceberg.catalog.Catalog sourceIcebergCatalog, + org.apache.iceberg.catalog.Catalog targetIcebergCatalog) { + // no namespaces to sync if catalog does not implement SupportsNamespaces + if (sourceIcebergCatalog instanceof SupportsNamespaces sourceNamespaceCatalog + && targetIcebergCatalog instanceof SupportsNamespaces targetNamespaceCatalog) { + List namespacesSource; + + try { + namespacesSource = sourceNamespaceCatalog.listNamespaces(parentNamespace); + clientLogger.info( + "Listed {} namespaces in namespace {} for catalog {} from source.", + namespacesSource.size(), + parentNamespace, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list namespaces in namespace {} for catalog {} from source.", + parentNamespace, + catalogName, + e); + return; + } + + List namespacesTarget; + + try { + namespacesTarget = targetNamespaceCatalog.listNamespaces(parentNamespace); + clientLogger.info( + "Listed {} namespaces in namespace {} for catalog {} from target.", + namespacesTarget.size(), + parentNamespace, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list namespaces in namespace {} for catalog {} from target.", + parentNamespace, + catalogName, + e); + return; + } + + SynchronizationPlan namespaceSynchronizationPlan = + syncPlanner.planNamespaceSync( + catalogName, parentNamespace, namespacesSource, namespacesTarget); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(namespaceSynchronizationPlan); + + namespaceSynchronizationPlan + .entitiesNotModified() + .forEach( + namespace -> + clientLogger.info( + "No change detected for namespace {} in namespace {} for catalog {}, skipping.", + namespace, + parentNamespace, + catalogName)); + + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToCreate()) { + try { + targetNamespaceCatalog.createNamespace(namespace); + clientLogger.info( + "Created namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to create namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToOverwrite()) { + try { + Map sourceNamespaceMetadata = + sourceNamespaceCatalog.loadNamespaceMetadata(namespace); + Map targetNamespaceMetadata = + targetNamespaceCatalog.loadNamespaceMetadata(namespace); + + if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) { + clientLogger.info( + "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + continue; + } + + target.dropNamespaceCascade(targetIcebergCatalog, namespace); + targetNamespaceCatalog.createNamespace(namespace, sourceNamespaceMetadata); + + clientLogger.info( + "Overwrote namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to overwrite namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToRemove()) { + try { + target.dropNamespaceCascade(targetIcebergCatalog, namespace); + clientLogger.info( + "Removed namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToSyncChildren()) { + syncTables(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog); + syncNamespaces(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog); + } + } + } + + /** + * Sync tables contained within a namespace. + * + * @param catalogName + * @param namespace + * @param sourceIcebergCatalog + * @param targetIcebergCatalog + */ + public void syncTables( + String catalogName, + Namespace namespace, + org.apache.iceberg.catalog.Catalog sourceIcebergCatalog, + org.apache.iceberg.catalog.Catalog targetIcebergCatalog) { + Set sourceTables; + + try { + sourceTables = new HashSet<>(sourceIcebergCatalog.listTables(namespace)); + clientLogger.info( + "Listed {} tables in namespace {} for catalog {} on source.", + sourceTables.size(), + namespace, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list tables in namespace {} for catalog {} on source.", + namespace, + catalogName, + e); + return; + } + + Set targetTables; + + try { + targetTables = new HashSet<>(targetIcebergCatalog.listTables(namespace)); + clientLogger.info( + "Listed {} tables in namespace {} for catalog {} on target.", + targetTables.size(), + namespace, + catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list tables in namespace {} for catalog {} on target.", + namespace, + catalogName, + e); + return; + } + + SynchronizationPlan tableSyncPlan = + syncPlanner.planTableSync(catalogName, namespace, sourceTables, targetTables); + + tableSyncPlan + .entitiesToSkip() + .forEach( + tableId -> + clientLogger.info( + "Skipping table {} in namespace {} in catalog {}.", + tableId, + namespace, + catalogName)); + + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(tableSyncPlan); + + for (TableIdentifier tableId : tableSyncPlan.entitiesToCreate()) { + try { + Table table = sourceIcebergCatalog.loadTable(tableId); + + if (table instanceof BaseTable baseTable) { + targetIcebergCatalog.registerTable( + tableId, baseTable.operations().current().metadataFileLocation()); + } else { + throw new IllegalStateException("Cannot register table that does not extend BaseTable."); + } + + if (table instanceof BaseTableWithETag tableWithETag) { + etagService.storeETag(catalogName, tableId, tableWithETag.etag()); + } + + clientLogger.info( + "Registered table {} in namespace {} in catalog {}. - {}/{}", + tableId, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to register table {} in namespace {} in catalog {}. - {}/{}", + tableId, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (TableIdentifier tableId : tableSyncPlan.entitiesToOverwrite()) { + try { + Table table; + + if (sourceIcebergCatalog instanceof PolarisCatalog polarisCatalog) { + String etag = etagService.getETag(catalogName, tableId); + table = polarisCatalog.loadTable(tableId, etag); + } else { + table = sourceIcebergCatalog.loadTable(tableId); + } + + if (table instanceof BaseTable baseTable) { + targetIcebergCatalog.dropTable(tableId, /* purge */ false); + targetIcebergCatalog.registerTable( + tableId, baseTable.operations().current().metadataFileLocation()); + } else { + throw new IllegalStateException("Cannot register table that does not extend BaseTable."); + } + + if (table instanceof BaseTableWithETag tableWithETag) { + etagService.storeETag(catalogName, tableId, tableWithETag.etag()); + } + + clientLogger.info( + "Dropped and re-registered table {} in namespace {} in catalog {}. - {}/{}", + tableId, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (NotModifiedException e) { + clientLogger.info( + "Table {} in namespace {} in catalog {} with was not modified, not overwriting in target catalog. - {}/{}", + tableId, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to drop and re-register table {} in namespace {} in catalog {}. - {}/{}", + tableId, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + + for (TableIdentifier table : tableSyncPlan.entitiesToRemove()) { + try { + targetIcebergCatalog.dropTable(table, /* purge */ false); + clientLogger.info( + "Dropped table {} in namespace {} in catalog {}. - {}/{}", + table, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.info( + "Failed to drop table {} in namespace {} in catalog {}. - {}/{}", + table, + namespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); + } + } + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java new file mode 100644 index 00000000..58be6a95 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.access; + +public class AccessControlConstants { + + public static final String OMNIPOTENCE_PROPERTY = "IS_OMNIPOTENT_PRINCIPAL"; + + protected static final String OMNIPOTENT_PRINCIPAL_NAME_PREFIX = "omnipotent-principal-"; +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java new file mode 100644 index 00000000..c0b4fe6e --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.access; + +import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_MANAGE_METADATA; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_READ_PROPERTIES; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_LIST; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_READ_PROPERTIES; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_LIST; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_READ_PROPERTIES; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_LIST; +import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_READ_PROPERTIES; +import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENCE_PROPERTY; +import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENT_PRINCIPAL_NAME_PREFIX; + +import java.util.List; +import java.util.NoSuchElementException; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.PolarisService; + +/** + * Service class to facilitate the access control needs of the synchronization. This involves + * setting up principals, principal roles, catalog roles, and grants to allow the tool to be able to + * introspect into catalog internals like catalog-roles, tables, grants. + */ +public class AccessControlService { + + private final PolarisService polaris; + + public AccessControlService(PolarisService polaris) { + this.polaris = polaris; + } + + /** + * Creates or replaces the existing omnipotent principal on the provided polaris instance. + * + * @param replace if true, if an existing omnipotent principal role exists, it will be dropped and + * recreated + * @return the principal and credentials for the omnipotent principal + */ + public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) { + List principals = polaris.listPrincipals(); + + Principal omnipotentPrincipalPrototype = + new Principal() + .name(OMNIPOTENT_PRINCIPAL_NAME_PREFIX + System.currentTimeMillis()) + .putPropertiesItem( + OMNIPOTENCE_PROPERTY, ""); // this property identifies the omnipotent principal + + for (Principal principal : principals) { + if (principal.getProperties() != null + && principal.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { + if (replace) { + // drop existing omnipotent principal in preparation for replacement + polaris.removePrincipal(principal.getName()); + } else { + // we cannot create another omnipotent principal and cannot replace the existing, fail + throw new IllegalStateException( + "Not permitted to replace existing omnipotent principal, but omnipotent " + + "principal with property " + + OMNIPOTENCE_PROPERTY + + " already exists"); + } + } + } + + // existing principal with identifying property does not exist, create a new one + return polaris.createPrincipal(omnipotentPrincipalPrototype, false); + } + + /** + * Retrieves the omnipotent principal role for the provided principalName. + * + * @param principalName the principal name to search for roles with + * @return the principal role for the provided principal, if exists + */ + public PrincipalRole getOmnipotentPrincipalRoleForPrincipal(String principalName) { + List principalRolesAssigned = + polaris.listPrincipalRolesAssignedForPrincipal(principalName); + + return principalRolesAssigned.stream() + .filter( + principalRole -> + principalRole.getProperties() != null + && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) + .findFirst() + .orElseThrow( + () -> + new NoSuchElementException( + "No omnipotent principal role with property " + + OMNIPOTENCE_PROPERTY + + " exists for principal " + + principalName)); + } + + /** + * Creates a principal role for the omnipotent principal and assigns it to the provided omnipotent + * principal. + * + * @param omnipotentPrincipal the principal to create and assign the role for + * @param replace if true, drops existing omnipotent principal roles if they exist before creating + * the new one + * @return the principal role for the omnipotent principal + */ + public PrincipalRole createAndAssignPrincipalRole( + PrincipalWithCredentials omnipotentPrincipal, boolean replace) { + List principalRoles = polaris.listPrincipalRoles(); + + PrincipalRole omnipotentPrincipalRole = + new PrincipalRole() + .name(omnipotentPrincipal.getPrincipal().getName()) + .putPropertiesItem(OMNIPOTENCE_PROPERTY, ""); + + for (PrincipalRole principalRole : principalRoles) { + if (principalRole.getProperties() != null + && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { + // replace existing principal role if exists + if (replace) { + polaris.removePrincipalRole(principalRole.getName()); + } else { + throw new IllegalStateException( + "Not permitted to replace existing omnipotent principal role, but omnipotent " + + "principal role with property " + + OMNIPOTENCE_PROPERTY + + " already exists"); + } + } + } + + polaris.createPrincipalRole(omnipotentPrincipalRole, false); + polaris.assignPrincipalRole( + omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName()); + return omnipotentPrincipalRole; + } + + /** + * Creates an omnipotent catalog role for a catalog and assigns it to the provided omnipotent + * principal role. + * + * @param catalogName the catalog to create the catalog role for + * @param omnipotentPrincipalRole the omnipotent principal role to assign the created catalog role + * to + * @param replace if true, drops and recreates the existing omnipotent catalog role + * @return the created omnipotent catalog role + */ + public CatalogRole createAndAssignCatalogRole( + String catalogName, PrincipalRole omnipotentPrincipalRole, boolean replace) { + List catalogRoles = polaris.listCatalogRoles(catalogName); + + for (CatalogRole catalogRole : catalogRoles) { + if (catalogRole.getProperties() != null + && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { + if (replace) { + polaris.removeCatalogRole(catalogName, catalogRole.getName()); + } else { + throw new IllegalStateException( + "Not permitted to replace existing omnipotent catalog role for catalog " + + catalogName + + ", but omnipotent principal with property " + + OMNIPOTENCE_PROPERTY + + " already exists"); + } + } + } + + CatalogRole omnipotentCatalogRole = + new CatalogRole() + .name(omnipotentPrincipalRole.getName()) + .putPropertiesItem(OMNIPOTENCE_PROPERTY, ""); + + polaris.createCatalogRole(catalogName, omnipotentCatalogRole, false /* overwrite */); + polaris.assignCatalogRole( + omnipotentPrincipalRole.getName(), catalogName, omnipotentCatalogRole.getName()); + return omnipotentCatalogRole; + } + + /** + * Adds grants for privilege level desired on the omnipotent catalog role. + * + * @param catalogName the catalog to identify the role in + * @param catalogRoleName the name of the catalog role to assign the grants tpo + * @param withWriteAccess if the catalog role should be given write access to the catalog + * internals + * @return the grants that were added to the catalog role + */ + public List addGrantsToCatalogRole( + String catalogName, String catalogRoleName, boolean withWriteAccess) { + if (withWriteAccess) { + // write access only requires CATALOG_MANAGE_METADATA + CatalogGrant catalogManageMetadata = + new CatalogGrant() + .type(GrantResource.TypeEnum.CATALOG) + .privilege(CATALOG_MANAGE_METADATA); + + polaris.addGrant(catalogName, catalogRoleName, catalogManageMetadata); + return List.of(catalogManageMetadata); + } else { + // read access requires reading properties and listing entities for each entity type + CatalogGrant catalogReadProperties = + new CatalogGrant() + .type(GrantResource.TypeEnum.CATALOG) + .privilege(CATALOG_READ_PROPERTIES); + + CatalogGrant namespaceReadProperties = + new CatalogGrant() + .type(GrantResource.TypeEnum.CATALOG) + .privilege(NAMESPACE_READ_PROPERTIES); + + CatalogGrant namespaceList = + new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(NAMESPACE_LIST); + + CatalogGrant tableReadProperties = + new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_READ_PROPERTIES); + + CatalogGrant tableList = + new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_LIST); + + CatalogGrant viewReadProperties = + new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_READ_PROPERTIES); + + CatalogGrant viewList = + new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_LIST); + + polaris.addGrant(catalogName, catalogRoleName, catalogReadProperties); + polaris.addGrant(catalogName, catalogRoleName, namespaceReadProperties); + polaris.addGrant(catalogName, catalogRoleName, namespaceList); + polaris.addGrant(catalogName, catalogRoleName, tableReadProperties); + polaris.addGrant(catalogName, catalogRoleName, tableList); + polaris.addGrant(catalogName, catalogRoleName, viewReadProperties); + polaris.addGrant(catalogName, catalogRoleName, viewList); + return List.of( + catalogReadProperties, namespaceReadProperties, tableReadProperties, viewReadProperties); + } + } + + /** + * Determines if an omnipotent catalog role already exists for this catalog. + * + * @param catalogName the catalog to search in + * @return true if exists, false otherwise + */ + public boolean omnipotentCatalogRoleExists(String catalogName) { + List catalogRoles = polaris.listCatalogRoles(catalogName); + + return catalogRoles.stream() + .anyMatch( + catalogRole -> + catalogRole.getProperties() != null + && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)); + } + + /** + * Creates catalog role for catalog, assigns it to provided principal role, and assigns grants + * with appropriate privilege level. + * + * @param catalogName the catalog to create the role for + * @param omnipotentPrincipalRole the principal role to assign the catalog role to + * @param replace if true, drops the existing catalog role if it exists + * @param withWriteAccess gives write access to the catalog role + */ + public void setupOmnipotentRoleForCatalog( + String catalogName, + PrincipalRole omnipotentPrincipalRole, + boolean replace, + boolean withWriteAccess) { + CatalogRole omniPotentCatalogRole = + createAndAssignCatalogRole(catalogName, omnipotentPrincipalRole, replace); + addGrantsToCatalogRole(catalogName, omniPotentCatalogRole.getName(), withWriteAccess); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java new file mode 100644 index 00000000..1f28a031 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.metrics.MetricsReporter; + +/** Wrapper around {@link BaseTable} that contains the latest ETag for the table. */ +public class BaseTableWithETag extends BaseTable { + + private final String etag; + + public BaseTableWithETag(TableOperations ops, String name, String etag) { + super(ops, name); + this.etag = etag; + } + + public BaseTableWithETag( + TableOperations ops, String name, MetricsReporter reporter, String etag) { + super(ops, name, reporter); + this.etag = etag; + } + + public String etag() { + return etag; + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java new file mode 100644 index 00000000..b06e4281 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import org.apache.iceberg.catalog.TableIdentifier; + +/** + * Generic interface to provide and store ETags for tables within catalogs. This allows the storage + * of the ETag to be completely independent from the tool. + */ +public interface ETagService { + + /** + * Retrieves the ETag for the table. + * + * @param catalogName the catalog the table is in + * @param tableIdentifier the table identifier + * @return The ETag for the last known metadata for the table + */ + String getETag(String catalogName, TableIdentifier tableIdentifier); + + /** + * After table loading, stores the fetched ETag. + * + * @param catalogName the catalog the table is in + * @param tableIdentifier the table identifier + * @param etag the ETag that was provided by the Iceberg REST api + */ + void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag); +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java new file mode 100644 index 00000000..8a673c94 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import java.util.NoSuchElementException; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; + +/** + * Wrapper table operations class that just allows fetching a provided table metadata. Used to build + * a {@link org.apache.iceberg.BaseTable} without having to expose a full-fledged operations class. + */ +public class MetadataWrapperTableOperations implements TableOperations { + + private final TableMetadata tableMetadata; + + public MetadataWrapperTableOperations(TableMetadata tableMetadata) { + this.tableMetadata = tableMetadata; + } + + @Override + public TableMetadata current() { + return this.tableMetadata; + } + + @Override + public TableMetadata refresh() { + return this.tableMetadata; + } + + @Override + public void commit(TableMetadata tableMetadata, TableMetadata tableMetadata1) { + throw new UnsupportedOperationException("Cannot perform commit."); + } + + @Override + public FileIO io() { + throw new NoSuchElementException("Does not possess file io."); + } + + @Override + public String metadataFileLocation(String s) { + return this.tableMetadata.metadataFileLocation(); + } + + @Override + public LocationProvider locationProvider() { + throw new NoSuchElementException("Does not possess location provider."); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java new file mode 100644 index 00000000..04f3bd0c --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import org.apache.iceberg.catalog.TableIdentifier; + +/** Implementation that returns nothing and stores no ETags. */ +public class NoOpETagService implements ETagService { + + @Override + public String getETag(String catalogName, TableIdentifier tableIdentifier) { + return null; + } + + @Override + public void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag) {} +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java new file mode 100644 index 00000000..6903c642 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import org.apache.iceberg.catalog.TableIdentifier; + +public class NotModifiedException extends RuntimeException { + + public NotModifiedException(TableIdentifier tableIdentifier) { + super("Table " + tableIdentifier + " was not modified."); + } + + public NotModifiedException(String message) { + super(message); + } + + public NotModifiedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java new file mode 100644 index 00000000..553d7253 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.catalog; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Map; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.ResourcePaths; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadTableResponseParser; +import org.apache.polaris.tools.sync.polaris.http.OAuth2Util; + +/** + * Overrides loadTable default implementation to issue a custom loadTable request to the Polaris + * Iceberg REST Api and build the table metadata. This is necessary since the existing {@link + * RESTCatalog} does not provide a way to capture response headers to retrieve the ETag on a + * loadTable request. + */ +public class PolarisCatalog extends RESTCatalog + implements Catalog, ViewCatalog, SupportsNamespaces, Configurable, Closeable { + + private String name = null; + + private Map properties = null; + + private String accessToken = null; + + private HttpClient httpClient = null; + + private ObjectMapper objectMapper = null; + + private ResourcePaths resourcePaths = null; + + public PolarisCatalog() { + super(); + } + + @Override + public void initialize(String name, Map props) { + this.name = name; + this.properties = props; + + if (resourcePaths == null) { + this.properties.put("prefix", props.get("warehouse")); + resourcePaths = ResourcePaths.forCatalogProperties(this.properties); + } + + if (accessToken == null || httpClient == null || this.objectMapper == null) { + String oauth2ServerUri = props.get("uri") + "/v1/oauth/tokens"; + String credential = props.get("credential"); + + String clientId = credential.split(":")[0]; + String clientSecret = credential.split(":")[1]; + + String scope = props.get("scope"); + + // TODO: Add token refresh + try { + this.accessToken = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope); + } catch (IOException e) { + throw new RuntimeException(e); + } + + this.httpClient = HttpClient.newBuilder().build(); + this.objectMapper = new ObjectMapper(); + } + super.initialize(name, props); + } + + @Override + public Table loadTable(TableIdentifier ident) { + return loadTable(ident, null); + } + + /** + * Perform a loadTable with a specified ETag in the If-None-Match header. TODO: Remove this once + * ETag is officially supported in Iceberg + * + * @param ident the identifier of the table + * @param etag the etag + * @return a {@link BaseTable} if no ETag was found in the response headers. A {@link + * BaseTableWithETag} if an ETag was included in the response headers. + * @throws NotModifiedException if the Iceberg REST catalog responded with 304 NOT MODIFIED + */ + public Table loadTable(TableIdentifier ident, String etag) { + String catalogName = this.properties.get("warehouse"); + + String tablePath = + String.format("%s/%s", this.properties.get("uri"), resourcePaths.table(ident)); + + HttpRequest.Builder requestBuilder = + HttpRequest.newBuilder() + .uri(URI.create(tablePath)) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken) + .GET(); + + // specify last known etag in if-none-match header + if (etag != null) { + requestBuilder.header(HttpHeaders.IF_NONE_MATCH, etag); + } + + HttpRequest request = requestBuilder.build(); + + HttpResponse response; + + try { + response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // api responded with 304 not modified, throw from here to signal + if (response.statusCode() == HttpStatus.SC_NOT_MODIFIED) { + throw new NotModifiedException(ident); + } + + String body = response.body(); + + String newETag = null; + + // if etag header is present in response, store new provided etag + if (response.headers().firstValue(HttpHeaders.ETAG).isPresent()) { + newETag = response.headers().firstValue(HttpHeaders.ETAG).get(); + } + + // build custom base table with metadata so that tool can retrieve the + // location and register it on the target side + LoadTableResponse loadTableResponse = LoadTableResponseParser.fromJson(body); + MetadataWrapperTableOperations ops = + new MetadataWrapperTableOperations(loadTableResponse.tableMetadata()); + + if (newETag != null) { + return new BaseTableWithETag(ops, CatalogUtil.fullTableName(catalogName, ident), newETag); + } + + return new BaseTable(ops, CatalogUtil.fullTableName(catalogName, ident)); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java new file mode 100644 index 00000000..50c1428d --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.http; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; + +/** Encapsulates handy http utility methods. */ +public class HttpUtil { + + /** Turn a {@link Map} into an xxx-url-form-encoded compatible String form body. */ + public static String constructFormEncodedString(Map parameters) { + return parameters.entrySet().stream() + .map( + entry -> + URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)) + .collect(Collectors.joining("&")); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java new file mode 100644 index 00000000..98ada786 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.http; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Map; +import java.util.NoSuchElementException; +import org.apache.http.HttpHeaders; +import org.apache.http.entity.ContentType; + +/** Utility class to manage OAuth2 flow for a Polaris instance. */ +public class OAuth2Util { + + private static final HttpClient httpClient = HttpClient.newHttpClient(); + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static String fetchToken( + String oauth2ServerUri, String clientId, String clientSecret, String scope) + throws IOException { + + Map formBody = + Map.of( + "grant_type", "client_credentials", + "scope", scope, + "client_id", clientId, + "client_secret", clientSecret); + + String formBodyAsString = HttpUtil.constructFormEncodedString(formBody); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(oauth2ServerUri)) + .header(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType()) + .POST(HttpRequest.BodyPublishers.ofString(formBodyAsString)) + .build(); + + try { + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + Map responseBody = + objectMapper.readValue(response.body(), new TypeReference<>() {}); + + String accessToken = responseBody.getOrDefault("access_token", null); + + if (accessToken != null) { + return accessToken; + } + + throw new NoSuchElementException( + "No field 'access_token' found in response from oauth2-server-uri."); + } catch (Exception e) { + throw new RuntimeException("Could not fetch access token", e); + } + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java new file mode 100644 index 00000000..0c6f4b10 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import java.util.ArrayList; +import java.util.List; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +/** + * Planner that filters out access control entities that should not be modified in the duration of + * the sync. This includes the omnipotent roles and principals that we do not want to copy between + * the two instances as well as modifications to service_admin or catalog_admin that may disrupt + * manage_access permissions. + */ +public class AccessControlAwarePlanner extends DelegatedPlanner implements SynchronizationPlanner { + + public AccessControlAwarePlanner(SynchronizationPlanner delegate) { + super(delegate); + } + + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + List skippedRoles = new ArrayList<>(); + List filteredRolesSource = new ArrayList<>(); + List filteredRolesTarget = new ArrayList<>(); + + for (PrincipalRole role : principalRolesOnSource) { + // filter out omnipotent principal role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out service_admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesSource.add(role); + } + + for (PrincipalRole role : principalRolesOnTarget) { + // filter out omnipotent principal role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out service admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesTarget.add(role); + } + + SynchronizationPlan delegatedPlan = + this.delegate.planPrincipalRoleSync(filteredRolesSource, filteredRolesTarget); + + for (PrincipalRole role : skippedRoles) { + delegatedPlan.skipEntity(role); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget) { + List skippedRoles = new ArrayList<>(); + List filteredRolesSource = new ArrayList<>(); + List filteredRolesTarget = new ArrayList<>(); + + for (CatalogRole role : catalogRolesOnSource) { + // filter out omnipotent catalog role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out catalog admin + if (role.getName().equals("catalog_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesSource.add(role); + } + + for (CatalogRole role : catalogRolesOnTarget) { + // filter out omnipotent catalog role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out catalog admin + if (role.getName().equals("catalog_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesTarget.add(role); + } + + SynchronizationPlan delegatedPlan = + this.delegate.planCatalogRoleSync(catalogName, filteredRolesSource, filteredRolesTarget); + + for (CatalogRole role : skippedRoles) { + delegatedPlan.skipEntityAndSkipChildren(role); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget) { + List skippedRoles = new ArrayList<>(); + List filteredRolesSource = new ArrayList<>(); + List filteredRolesTarget = new ArrayList<>(); + + for (PrincipalRole role : assignedPrincipalRolesOnSource) { + // filter out assignment to omnipotent catalog role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out assignment to service admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesSource.add(role); + } + + for (PrincipalRole role : assignedPrincipalRolesOnTarget) { + // filer out assignment to omnipotent principal role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out assignment to service admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesTarget.add(role); + } + + SynchronizationPlan delegatedPlan = + this.delegate.planAssignPrincipalRolesToCatalogRolesSync( + catalogName, catalogRoleName, filteredRolesSource, filteredRolesTarget); + + for (PrincipalRole role : skippedRoles) { + delegatedPlan.skipEntity(role); + } + + return delegatedPlan; + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java new file mode 100644 index 00000000..e48b5323 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import java.util.List; +import java.util.Set; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +/** + * Extend this to delegate planning to another planner, but only override methods for the + * functionality needed. + */ +public abstract class DelegatedPlanner implements SynchronizationPlanner { + + protected final SynchronizationPlanner delegate; + + public DelegatedPlanner(SynchronizationPlanner delegate) { + this.delegate = delegate; + } + + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + return delegate.planPrincipalRoleSync(principalRolesOnSource, principalRolesOnTarget); + } + + @Override + public SynchronizationPlan planCatalogSync( + List catalogsOnSource, List catalogsOnTarget) { + return delegate.planCatalogSync(catalogsOnSource, catalogsOnTarget); + } + + @Override + public SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget) { + return delegate.planCatalogRoleSync(catalogName, catalogRolesOnSource, catalogRolesOnTarget); + } + + @Override + public SynchronizationPlan planGrantSync( + String catalogName, + String catalogRoleName, + List grantsOnSource, + List grantsOnTarget) { + return delegate.planGrantSync(catalogName, catalogRoleName, grantsOnSource, grantsOnTarget); + } + + @Override + public SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget) { + return delegate.planAssignPrincipalRolesToCatalogRolesSync( + catalogName, + catalogRoleName, + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget); + } + + @Override + public SynchronizationPlan planNamespaceSync( + String catalogName, + Namespace namespace, + List namespacesOnSource, + List namespacesOnTarget) { + return delegate.planNamespaceSync( + catalogName, namespace, namespacesOnSource, namespacesOnTarget); + } + + @Override + public SynchronizationPlan planTableSync( + String catalogName, + Namespace namespace, + Set tablesOnSource, + Set tablesOnTarget) { + return delegate.planTableSync(catalogName, namespace, tablesOnSource, tablesOnTarget); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java new file mode 100644 index 00000000..67a38a21 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +/** Planner that checks for modifications and plans to skip entities that have not been modified. */ +public class ModificationAwarePlanner implements SynchronizationPlanner { + + private static final String CREATE_TIMESTAMP = "createTimestamp"; + + private static final String LAST_UPDATE_TIMESTAMP = "lastUpdateTimestamp"; + + private static final String ENTITY_VERSION = "entityVersion"; + + private static final List DEFAULT_KEYS_TO_IGNORE = + List.of(CREATE_TIMESTAMP, LAST_UPDATE_TIMESTAMP, ENTITY_VERSION); + + private static final List CATALOG_KEYS_TO_IGNORE = + List.of( + // defaults + CREATE_TIMESTAMP, + LAST_UPDATE_TIMESTAMP, + ENTITY_VERSION, + + // For certain storageConfigInfo fields, depending on the credentials Polaris was set up + // with + // to access the storage, some fields will always be different across the source and the + // target. + // For example, for S3 my source and target Polaris instances may be set up with different + // AWS users, + // each of which assumes the same role to access the storage + + // S3 + "storageConfigInfo.userArn", + + // AZURE + "storageConfigInfo.consentUrl", + "storageConfigInfo.multiTenantAppName", + + // GCP + "storageConfigInfo.gcsServiceAccount"); + + private final SynchronizationPlanner delegate; + + private final ObjectMapper objectMapper; + + public ModificationAwarePlanner(SynchronizationPlanner delegate) { + this.objectMapper = new ObjectMapper(); + this.delegate = delegate; + } + + /** + * Removes keys from the provided map. + * + * @param map the map to remove the keys from + * @param keysToRemove a list of keys, nested keys should be separated by '.' eg. "key1.key2" + * @return the map with the keys removed + */ + private Map removeKeys(Map map, List keysToRemove) { + Map cleaned = + objectMapper.convertValue(map, new TypeReference>() {}); + + for (String key : keysToRemove) { + // splits key into first part and rest, eg. key1.key2.key3 becomes [key1, key2.key3] + String[] separateFirst = key.split("\\.", 2); + String primary = separateFirst[0]; + + if (separateFirst.length > 1) { + // if there are more nested keys, we want to recursively search the sub map if it exists + Object valueForPrimary = cleaned.get(primary); // get object for primary key if it exists + + if (valueForPrimary == null) { + continue; + } + + try { + Map subMap = + objectMapper.convertValue(valueForPrimary, new TypeReference<>() {}); + Map cleanedSubMap = + removeKeys(subMap, List.of(separateFirst[1])); // remove nested keys from submap + cleaned.put(primary, cleanedSubMap); // replace sub-map with key removed + } catch (IllegalArgumentException e) { + // do nothing because that means the key does not exist, no need to remove it + } + } else { + cleaned.remove(primary); // just remove the key if we have no more nesting + } + } + + return cleaned; + } + + /** + * Compares two objects to see if they are the same. + * + * @param o1 + * @param o2 + * @param keysToIgnore list of keys to ignore in the comparison + * @return true if they are the same, false otherwise + */ + private boolean areSame(Object o1, Object o2, List keysToIgnore) { + Map o1AsMap = objectMapper.convertValue(o1, new TypeReference<>() {}); + Map o2AsMap = objectMapper.convertValue(o2, new TypeReference<>() {}); + o1AsMap = removeKeys(o1AsMap, keysToIgnore); + o2AsMap = removeKeys(o2AsMap, keysToIgnore); + return o1AsMap.equals(o2AsMap); + } + + private boolean areSame(Object o1, Object o2) { + return areSame(o1, o2, DEFAULT_KEYS_TO_IGNORE); + } + + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + Map sourceRolesByName = new HashMap<>(); + Map targetRolesByName = new HashMap<>(); + + List notModifiedPrincipalRoles = new ArrayList<>(); + + principalRolesOnSource.forEach(role -> sourceRolesByName.put(role.getName(), role)); + principalRolesOnTarget.forEach(role -> targetRolesByName.put(role.getName(), role)); + + for (PrincipalRole sourceRole : principalRolesOnSource) { + if (targetRolesByName.containsKey(sourceRole.getName())) { + PrincipalRole targetRole = targetRolesByName.get(sourceRole.getName()); + + if (areSame(sourceRole, targetRole)) { + targetRolesByName.remove(targetRole.getName()); + sourceRolesByName.remove(sourceRole.getName()); + notModifiedPrincipalRoles.add(sourceRole); + } + } + } + + SynchronizationPlan delegatedPlan = + delegate.planPrincipalRoleSync( + sourceRolesByName.values().stream().toList(), + targetRolesByName.values().stream().toList()); + + for (PrincipalRole principalRole : notModifiedPrincipalRoles) { + delegatedPlan.skipEntityNotModified(principalRole); + } + + return delegatedPlan; + } + + private boolean areSame(Catalog source, Catalog target) { + return areSame(source, target, CATALOG_KEYS_TO_IGNORE) + // because of the way the jackson serialization works, any class that extends HashMap is + // serialized + // with just the fields in the map. Unfortunately, CatalogProperties extends HashMap so we + // must + // manually compare the fields in the catalog properties and cannot automatically + // deserialize them + // as a map + && Objects.equals(source.getProperties(), target.getProperties()); + } + + @Override + public SynchronizationPlan planCatalogSync( + List catalogsOnSource, List catalogsOnTarget) { + Map sourceCatalogsByName = new HashMap<>(); + Map targetCatalogsByName = new HashMap<>(); + + List notModifiedCatalogs = new ArrayList<>(); + + catalogsOnSource.forEach(catalog -> sourceCatalogsByName.put(catalog.getName(), catalog)); + catalogsOnTarget.forEach(catalog -> targetCatalogsByName.put(catalog.getName(), catalog)); + + for (Catalog sourceCatalog : catalogsOnSource) { + if (targetCatalogsByName.containsKey(sourceCatalog.getName())) { + Catalog targetCatalog = targetCatalogsByName.get(sourceCatalog.getName()); + + if (areSame(sourceCatalog, targetCatalog)) { + targetCatalogsByName.remove(targetCatalog.getName()); + sourceCatalogsByName.remove(sourceCatalog.getName()); + notModifiedCatalogs.add(sourceCatalog); + } + } + } + + SynchronizationPlan delegatedPlan = + delegate.planCatalogSync( + sourceCatalogsByName.values().stream().toList(), + targetCatalogsByName.values().stream().toList()); + + for (Catalog catalog : notModifiedCatalogs) { + delegatedPlan.skipEntityNotModified(catalog); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget) { + Map sourceCatalogRolesByName = new HashMap<>(); + Map targetCatalogRolesByName = new HashMap<>(); + + List notModifiedCatalogRoles = new ArrayList<>(); + + catalogRolesOnSource.forEach(role -> sourceCatalogRolesByName.put(role.getName(), role)); + catalogRolesOnTarget.forEach(role -> targetCatalogRolesByName.put(role.getName(), role)); + + for (CatalogRole sourceCatalogRole : catalogRolesOnSource) { + if (targetCatalogRolesByName.containsKey(sourceCatalogRole.getName())) { + CatalogRole targetCatalogRole = targetCatalogRolesByName.get(sourceCatalogRole.getName()); + + if (areSame(sourceCatalogRole, targetCatalogRole)) { + targetCatalogRolesByName.remove(targetCatalogRole.getName()); + sourceCatalogRolesByName.remove(sourceCatalogRole.getName()); + notModifiedCatalogRoles.add(sourceCatalogRole); + } + } + } + + SynchronizationPlan delegatedPlan = + delegate.planCatalogRoleSync( + catalogName, + sourceCatalogRolesByName.values().stream().toList(), + targetCatalogRolesByName.values().stream().toList()); + + for (CatalogRole catalogRole : notModifiedCatalogRoles) { + delegatedPlan.skipEntityNotModified(catalogRole); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planGrantSync( + String catalogName, + String catalogRoleName, + List grantsOnSource, + List grantsOnTarget) { + Set sourceGrants = new HashSet<>(grantsOnSource); + Set targetGrants = new HashSet<>(grantsOnTarget); + + List notModifiedGrants = new ArrayList<>(); + + for (GrantResource grantResource : grantsOnSource) { + if (targetGrants.contains(grantResource)) { + sourceGrants.remove(grantResource); + targetGrants.remove(grantResource); + notModifiedGrants.add(grantResource); + } + } + + SynchronizationPlan delegatedPlan = + delegate.planGrantSync( + catalogName, + catalogRoleName, + sourceGrants.stream().toList(), + targetGrants.stream().toList()); + + for (GrantResource grant : notModifiedGrants) { + delegatedPlan.skipEntityNotModified(grant); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget) { + return delegate.planAssignPrincipalRolesToCatalogRolesSync( + catalogName, + catalogRoleName, + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget); + } + + @Override + public SynchronizationPlan planNamespaceSync( + String catalogName, + Namespace namespace, + List namespacesOnSource, + List namespacesOnTarget) { + return delegate.planNamespaceSync( + catalogName, namespace, namespacesOnSource, namespacesOnTarget); + } + + @Override + public SynchronizationPlan planTableSync( + String catalogName, + Namespace namespace, + Set tablesOnSource, + Set tablesOnTarget) { + return delegate.planTableSync(catalogName, namespace, tablesOnSource, tablesOnTarget); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java new file mode 100644 index 00000000..241febd0 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import java.util.List; +import java.util.Set; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +public class NoOpSyncPlanner implements SynchronizationPlanner { + + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + return new SynchronizationPlan<>(); + } + + @Override + public SynchronizationPlan planCatalogSync( + List catalogsOnSource, List catalogsOnTarget) { + return new SynchronizationPlan<>(); + } + + @Override + public SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget) { + return new SynchronizationPlan<>(); + } + + @Override + public SynchronizationPlan planGrantSync( + String catalogName, + String catalogRoleName, + List grantsOnSource, + List grantsOnTarget) { + return new SynchronizationPlan<>(); + } + + @Override + public SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget) { + return new SynchronizationPlan<>(); + } + + @Override + public SynchronizationPlan planNamespaceSync( + String catalogName, + Namespace namespace, + List namespacesOnSource, + List namespacesOnTarget) { + return null; + } + + @Override + public SynchronizationPlan planTableSync( + String catalogName, + Namespace namespace, + Set tablesOnSource, + Set tablesOnTarget) { + return new SynchronizationPlan<>(); + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java new file mode 100644 index 00000000..622bfe25 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +/** + * Sync planner that attempts to create total parity between the source and target Polaris + * instances. This involves creating new entities, overwriting entities that exist on both source + * and target, and removing entities that exist only on the target. + */ +public class SourceParitySynchronizationPlanner implements SynchronizationPlanner { + + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + Set sourcePrincipalRoleNames = + principalRolesOnSource.stream().map(PrincipalRole::getName).collect(Collectors.toSet()); + Set targetPrincipalRoleNames = + principalRolesOnTarget.stream().map(PrincipalRole::getName).collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (PrincipalRole principalRole : principalRolesOnSource) { + if (targetPrincipalRoleNames.contains(principalRole.getName())) { + // overwrite roles that exist on both + plan.overwriteEntity(principalRole); + } else { + // create roles on target that only exist on source + plan.createEntity(principalRole); + } + } + + // remove roles that aren't on source + for (PrincipalRole principalRole : principalRolesOnTarget) { + if (!sourcePrincipalRoleNames.contains(principalRole.getName())) { + plan.removeEntity(principalRole); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planCatalogSync( + List catalogsOnSource, List catalogsOnTarget) { + Set sourceCatalogNames = + catalogsOnSource.stream().map(Catalog::getName).collect(Collectors.toSet()); + Set targetCatalogNames = + catalogsOnTarget.stream().map(Catalog::getName).collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (Catalog catalog : catalogsOnSource) { + if (targetCatalogNames.contains(catalog.getName())) { + // overwrite catalogs on target that exist on both + plan.overwriteEntity(catalog); + } else { + // create catalogs on target that exist only on source + plan.createEntity(catalog); + } + } + + // remove catalogs that are only on target + for (Catalog catalog : catalogsOnTarget) { + if (!sourceCatalogNames.contains(catalog.getName())) { + plan.removeEntity(catalog); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget) { + Set sourceCatalogRoleNames = + catalogRolesOnSource.stream().map(CatalogRole::getName).collect(Collectors.toSet()); + Set targetCatalogRoleNames = + catalogRolesOnTarget.stream().map(CatalogRole::getName).collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (CatalogRole catalogRole : catalogRolesOnSource) { + if (targetCatalogRoleNames.contains(catalogRole.getName())) { + plan.overwriteEntity(catalogRole); + // overwrite catalog roles on both + } else { + // create catalog roles on target that are only on source + plan.createEntity(catalogRole); + } + } + + // remove catalog roles on both the source and target + for (CatalogRole catalogRole : catalogRolesOnTarget) { + if (!sourceCatalogRoleNames.contains(catalogRole.getName())) { + plan.removeEntity(catalogRole); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planGrantSync( + String catalogName, + String catalogRoleName, + List grantsOnSource, + List grantsOnTarget) { + Set grantsSourceSet = Set.copyOf(grantsOnSource); + Set grantsTargetSet = Set.copyOf(grantsOnTarget); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + // special case: no concept of overwriting a grant + // it exists and cannot change, so just create new ones + for (GrantResource grant : grantsOnSource) { + if (!grantsTargetSet.contains(grant)) { + plan.createEntity(grant); + } + } + + // remove grants that are not on the source + for (GrantResource grant : grantsOnTarget) { + if (!grantsSourceSet.contains(grant)) { + plan.removeEntity(grant); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget) { + Set sourcePrincipalRoleNames = + assignedPrincipalRolesOnSource.stream() + .map(PrincipalRole::getName) + .collect(Collectors.toSet()); + Set targetPrincipalRoleNames = + assignedPrincipalRolesOnTarget.stream() + .map(PrincipalRole::getName) + .collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + // special case: no concept of overwriting an assignment of principal role to catalog role + // it either exists or it doesn't, it cannot change + for (PrincipalRole principalRole : assignedPrincipalRolesOnSource) { + if (!targetPrincipalRoleNames.contains(principalRole.getName())) { + plan.createEntity(principalRole); + } + } + + // revoke principal roles that do not exist on the source + for (PrincipalRole principalRole : assignedPrincipalRolesOnTarget) { + if (!sourcePrincipalRoleNames.contains(principalRole.getName())) { + plan.removeEntity(principalRole); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planNamespaceSync( + String catalogName, + Namespace namespace, + List namespacesOnSource, + List namespacesOnTarget) { + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (Namespace ns : namespacesOnSource) { + if (namespacesOnTarget.contains(ns)) { + // overwrite the entity on the target with the entity on the source + plan.overwriteEntity(ns); + } else { + // if the namespace is not on the target, plan to create it + plan.createEntity(ns); + } + } + + for (Namespace ns : namespacesOnTarget) { + if (!namespacesOnSource.contains(ns)) { + // remove namespaces that do not exist on the source but do exist on the target + plan.removeEntity(ns); + } + } + + return plan; + } + + @Override + public SynchronizationPlan planTableSync( + String catalogName, + Namespace namespace, + Set tablesOnSource, + Set tablesOnTarget) { + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (TableIdentifier tableIdentifier : tablesOnSource) { + if (tablesOnTarget.contains(tableIdentifier)) { + // overwrite tables on target and source + plan.overwriteEntity(tableIdentifier); + } else { + // create tables on source but not target + plan.createEntity(tableIdentifier); + } + } + + // remove tables only on target + for (TableIdentifier tableIdentifier : tablesOnTarget) { + if (!tablesOnSource.contains(tableIdentifier)) { + plan.removeEntity(tableIdentifier); + } + } + + return plan; + } +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java new file mode 100644 index 00000000..78edf9df --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning; + +import java.util.List; +import java.util.Set; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; + +/** + * Generic interface to generate synchronization plans for different types of entities based on what + * principal roles exist on the source and target. + */ +public interface SynchronizationPlanner { + + SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget); + + SynchronizationPlan planCatalogSync( + List catalogsOnSource, List catalogsOnTarget); + + SynchronizationPlan planCatalogRoleSync( + String catalogName, + List catalogRolesOnSource, + List catalogRolesOnTarget); + + SynchronizationPlan planGrantSync( + String catalogName, + String catalogRoleName, + List grantsOnSource, + List grantsOnTarget); + + SynchronizationPlan planAssignPrincipalRolesToCatalogRolesSync( + String catalogName, + String catalogRoleName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget); + + SynchronizationPlan planNamespaceSync( + String catalogName, + Namespace namespace, + List namespacesOnSource, + List namespacesOnTarget); + + SynchronizationPlan planTableSync( + String catalogName, + Namespace namespace, + Set tablesOnSource, + Set tablesOnTarget); +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java new file mode 100644 index 00000000..c5b5e500 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning.plan; + +public enum PlannedAction { + + /** For entities that are being freshly created on target. */ + CREATE, + + /** For entities that have to be dropped and recreated on target. */ + OVERWRITE, + + /** For entities that need to be dropped from the target. */ + REMOVE, + + /** + * For entities that should be skipped. Note that their child entities will still be synced. For + * example, we may skip a catalog role but its grants and assignments to principal roles will + * still be synced. + */ + SKIP, + + /** + * For entities that should be skipped due to no modification detected. Note that their child + * entities will still be synced. For example, we may skip a catalog role but its grants and + * assignments to principal roles will still be synced. + */ + SKIP_NOT_MODIFIED, + + /** + * For entities that should be skipped along with also skipping their child entities. Used in + * cases where we don't want to mess with an entire entity tree. For example we may not want to + * edit the catalog roles assigned to the service_admin. + */ + SKIP_AND_SKIP_CHILDREN +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java new file mode 100644 index 00000000..2153fc91 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.planning.plan; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Data structure that holds the state of all the planned modifications that should be made on the + * target. + * + * @param the entity type that the plan is for + */ +public class SynchronizationPlan { + + private final Map> entitiesForAction; + + public SynchronizationPlan() { + this.entitiesForAction = new HashMap<>(); + + for (PlannedAction action : PlannedAction.values()) { + this.entitiesForAction.put(action, new ArrayList<>()); + } + } + + public List entitiesForAction(PlannedAction action) { + return entitiesForAction.get(action); + } + + public List entitiesToCreate() { + return entitiesForAction(PlannedAction.CREATE); + } + + public List entitiesToOverwrite() { + return entitiesForAction(PlannedAction.OVERWRITE); + } + + public List entitiesToRemove() { + return entitiesForAction(PlannedAction.REMOVE); + } + + public List entitiesToSkip() { + return entitiesForAction(PlannedAction.SKIP); + } + + public List entitiesNotModified() { + return entitiesForAction(PlannedAction.SKIP_NOT_MODIFIED); + } + + public List entitiesToSkipAndSkipChildren() { + return entitiesForAction(PlannedAction.SKIP_AND_SKIP_CHILDREN); + } + + public List entitiesToSyncChildren() { + List entities = new ArrayList<>(); + + for (PlannedAction action : PlannedAction.values()) { + if (action != PlannedAction.SKIP_AND_SKIP_CHILDREN && action != PlannedAction.REMOVE) { + entities.addAll(entitiesForAction(action)); + } + } + + return entities; + } + + public void actOnEntity(PlannedAction action, T entity) { + this.entitiesForAction.get(action).add(entity); + } + + public void createEntity(T entity) { + this.actOnEntity(PlannedAction.CREATE, entity); + } + + public void overwriteEntity(T entity) { + this.actOnEntity(PlannedAction.OVERWRITE, entity); + } + + public void removeEntity(T entity) { + this.actOnEntity(PlannedAction.REMOVE, entity); + } + + public void skipEntity(T entity) { + this.actOnEntity(PlannedAction.SKIP, entity); + } + + public void skipEntityNotModified(T entity) { + this.actOnEntity(PlannedAction.SKIP_NOT_MODIFIED, entity); + } + + public void skipEntityAndSkipChildren(T entity) { + this.actOnEntity(PlannedAction.SKIP_AND_SKIP_CHILDREN, entity); + } +} diff --git a/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml b/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml new file mode 100644 index 00000000..ecf374c7 --- /dev/null +++ b/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml @@ -0,0 +1,1432 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +openapi: 3.0.3 +info: + title: Polaris Management Service + version: 0.0.1 + description: + Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals +servers: + - url: "{scheme}://{host}/api/management/v1" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost +# All routes are currently configured using an Authorization header. +security: + - OAuth2: [] + +paths: + /catalogs: + get: + operationId: listCatalogs + description: List all catalogs in this polaris service + responses: + 200: + description: List of catalogs in the polaris service + content: + application/json: + schema: + $ref: "#/components/schemas/Catalogs" + 403: + description: "The caller does not have permission to list catalog details" + post: + operationId: createCatalog + description: Add a new Catalog + requestBody: + description: The Catalog to create + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCatalogRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The caller does not have permission to create a catalog" + 404: + description: "The catalog does not exist" + 409: + description: "A catalog with the specified name already exists" + + /catalogs/{catalogName}: + parameters: + - name: catalogName + in: path + description: The name of the catalog + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: getCatalog + description: Get the details of a catalog + responses: + 200: + description: The catalog details + content: + application/json: + schema: + $ref: "#/components/schemas/Catalog" + 403: + description: "The caller does not have permission to read catalog details" + 404: + description: "The catalog does not exist" + + put: + operationId: updateCatalog + description: Update an existing catalog + requestBody: + description: The catalog details to use in the update + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateCatalogRequest" + responses: + 200: + description: The catalog details + content: + application/json: + schema: + $ref: "#/components/schemas/Catalog" + 403: + description: "The caller does not have permission to update catalog details" + 404: + description: "The catalog does not exist" + 409: + description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version" + + delete: + operationId: deleteCatalog + description: Delete an existing catalog. The catalog must be empty. + responses: + 204: + description: "Success, no content" + 403: + description: "The caller does not have permission to delete a catalog" + 404: + description: "The catalog does not exist" + + /principals: + get: + operationId: listPrincipals + description: List the principals for the current catalog + responses: + 200: + description: List of principals for this catalog + content: + application/json: + schema: + $ref: "#/components/schemas/Principals" + 403: + description: "The caller does not have permission to list catalog admins" + 404: + description: "The catalog does not exist" + + post: + operationId: createPrincipal + description: Create a principal + requestBody: + description: The principal to create + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreatePrincipalRequest" + responses: + 201: + description: "Successful response" + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalWithCredentials" + 403: + description: "The caller does not have permission to add a principal" + + /principals/{principalName}: + parameters: + - name: principalName + in: path + description: The principal name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: getPrincipal + description: Get the principal details + responses: + 200: + description: The requested principal + content: + application/json: + schema: + $ref: "#/components/schemas/Principal" + 403: + description: "The caller does not have permission to get principal details" + 404: + description: "The catalog or principal does not exist" + + put: + operationId: updatePrincipal + description: Update an existing principal + requestBody: + description: The principal details to use in the update + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdatePrincipalRequest" + responses: + 200: + description: The updated principal + content: + application/json: + schema: + $ref: "#/components/schemas/Principal" + 403: + description: "The caller does not have permission to update principal details" + 404: + description: "The principal does not exist" + 409: + description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version" + + delete: + operationId: deletePrincipal + description: Remove a principal from polaris + responses: + 204: + description: "Success, no content" + 403: + description: "The caller does not have permission to delete a principal" + 404: + description: "The principal does not exist" + + /principals/{principalName}/rotate: + parameters: + - name: principalName + in: path + description: The user name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + post: + operationId: rotateCredentials + description: Rotate a principal's credentials. The new credentials will be returned in the response. This is the only + API, aside from createPrincipal, that returns the user's credentials. This API is *not* idempotent. + responses: + 200: + description: The principal details along with the newly rotated credentials + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalWithCredentials" + 403: + description: "The caller does not have permission to rotate credentials" + 404: + description: "The principal does not exist" + + /principals/{principalName}/principal-roles: + parameters: + - name: principalName + in: path + description: The name of the target principal + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listPrincipalRolesAssigned + description: List the roles assigned to the principal + responses: + 200: + description: List of roles assigned to this principal + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalRoles" + 403: + description: "The caller does not have permission to list roles" + 404: + description: "The principal or catalog does not exist" + + put: + operationId: assignPrincipalRole + description: Add a role to the principal + requestBody: + description: The principal role to assign + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/GrantPrincipalRoleRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The caller does not have permission to add assign a role to the principal" + 404: + description: "The catalog, the principal, or the role does not exist" + + /principals/{principalName}/principal-roles/{principalRoleName}: + parameters: + - name: principalName + in: path + description: The name of the target principal + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: principalRoleName + in: path + description: The name of the role + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + delete: + operationId: revokePrincipalRole + description: Remove a role from a catalog principal + responses: + 204: + description: "Success, no content" + 403: + description: "The caller does not have permission to remove a role from the principal" + 404: + description: "The catalog or principal does not exist" + + /principal-roles: + get: + operationId: listPrincipalRoles + description: List the principal roles + responses: + 200: + description: List of principal roles + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalRoles" + 403: + description: "The caller does not have permission to list principal roles" + 404: + description: "The catalog does not exist" + + post: + operationId: createPrincipalRole + description: Create a principal role + requestBody: + description: The principal to create + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreatePrincipalRoleRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The caller does not have permission to add a principal role" + + /principal-roles/{principalRoleName}: + parameters: + - name: principalRoleName + in: path + description: The principal role name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: getPrincipalRole + description: Get the principal role details + responses: + 200: + description: The requested principal role + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalRole" + 403: + description: "The caller does not have permission to get principal role details" + 404: + description: "The principal role does not exist" + + put: + operationId: updatePrincipalRole + description: Update an existing principalRole + requestBody: + description: The principalRole details to use in the update + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/UpdatePrincipalRoleRequest" + responses: + 200: + description: The updated principal role + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalRole" + 403: + description: "The caller does not have permission to update principal role details" + 404: + description: "The principal role does not exist" + 409: + description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version" + + delete: + operationId: deletePrincipalRole + description: Remove a principal role from polaris + responses: + 204: + description: "Success, no content" + 403: + description: "The caller does not have permission to delete a principal role" + 404: + description: "The principal role does not exist" + + /principal-roles/{principalRoleName}/principals: + parameters: + - name: principalRoleName + in: path + description: The principal role name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listAssigneePrincipalsForPrincipalRole + description: List the Principals to whom the target principal role has been assigned + responses: + 200: + description: List the Principals to whom the target principal role has been assigned + content: + application/json: + schema: + $ref: "#/components/schemas/Principals" + 403: + description: "The caller does not have permission to list principals" + 404: + description: "The principal role does not exist" + + /principal-roles/{principalRoleName}/catalog-roles/{catalogName}: + parameters: + - name: principalRoleName + in: path + description: The principal role name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogName + in: path + required: true + description: The name of the catalog where the catalogRoles reside + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listCatalogRolesForPrincipalRole + description: Get the catalog roles mapped to the principal role + responses: + 200: + description: The list of catalog roles mapped to the principal role + content: + application/json: + schema: + $ref: "#/components/schemas/CatalogRoles" + 403: + description: "The caller does not have permission to list catalog roles" + 404: + description: "The principal role does not exist" + + put: + operationId: assignCatalogRoleToPrincipalRole + description: Assign a catalog role to a principal role + requestBody: + description: The principal to create + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/GrantCatalogRoleRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The caller does not have permission to assign a catalog role" + + /principal-roles/{principalRoleName}/catalog-roles/{catalogName}/{catalogRoleName}: + parameters: + - name: principalRoleName + in: path + description: The principal role name + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogName + in: path + description: The name of the catalog that contains the role to revoke + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogRoleName + in: path + description: The name of the catalog role that should be revoked + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + delete: + operationId: revokeCatalogRoleFromPrincipalRole + description: Remove a catalog role from a principal role + responses: + 204: + description: "Success, no content" + 403: + description: "The caller does not have permission to revoke a catalog role" + 404: + description: "The principal role does not exist" + + /catalogs/{catalogName}/catalog-roles: + parameters: + - name: catalogName + in: path + description: The catalog for which we are reading/updating roles + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listCatalogRoles + description: List existing roles in the catalog + responses: + 200: + description: The list of roles that exist in this catalog + content: + application/json: + schema: + $ref: "#/components/schemas/CatalogRoles" + post: + operationId: createCatalogRole + description: Create a new role in the catalog + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateCatalogRoleRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The principal is not authorized to create roles" + 404: + description: "The catalog does not exist" + + /catalogs/{catalogName}/catalog-roles/{catalogRoleName}: + parameters: + - name: catalogName + in: path + description: The catalog for which we are retrieving roles + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogRoleName + in: path + description: The name of the role + required: true + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: getCatalogRole + description: Get the details of an existing role + responses: + 200: + description: The specified role details + content: + application/json: + schema: + $ref: "#/components/schemas/CatalogRole" + 403: + description: "The principal is not authorized to read role data" + 404: + description: "The catalog or the role does not exist" + + put: + operationId: updateCatalogRole + description: Update an existing role in the catalog + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateCatalogRoleRequest" + responses: + 200: + description: The specified role details + content: + application/json: + schema: + $ref: "#/components/schemas/CatalogRole" + 403: + description: "The principal is not authorized to update roles" + 404: + description: "The catalog or the role does not exist" + 409: + description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version" + + delete: + operationId: deleteCatalogRole + description: Delete an existing role from the catalog. All associated grants will also be deleted + responses: + 204: + description: "Success, no content" + 403: + description: "The principal is not authorized to delete roles" + 404: + description: "The catalog or the role does not exist" + + /catalogs/{catalogName}/catalog-roles/{catalogRoleName}/principal-roles: + parameters: + - name: catalogName + in: path + required: true + description: The name of the catalog where the catalog role resides + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogRoleName + in: path + required: true + description: The name of the catalog role + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listAssigneePrincipalRolesForCatalogRole + description: List the PrincipalRoles to which the target catalog role has been assigned + responses: + 200: + description: List the PrincipalRoles to which the target catalog role has been assigned + content: + application/json: + schema: + $ref: "#/components/schemas/PrincipalRoles" + 403: + description: "The caller does not have permission to list principal roles" + 404: + description: "The catalog or catalog role does not exist" + + /catalogs/{catalogName}/catalog-roles/{catalogRoleName}/grants: + parameters: + - name: catalogName + in: path + required: true + description: The name of the catalog where the role will receive the grant + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + - name: catalogRoleName + in: path + required: true + description: The name of the role receiving the grant (must exist) + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + get: + operationId: listGrantsForCatalogRole + description: List the grants the catalog role holds + responses: + 200: + description: List of all grants given to the role in this catalog + content: + application/json: + schema: + $ref: "#/components/schemas/GrantResources" + put: + operationId: addGrantToCatalogRole + description: Add a new grant to the catalog role + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddGrantRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The principal is not authorized to create grants" + 404: + description: "The catalog or the role does not exist" + post: + operationId: revokeGrantFromCatalogRole + description: + Delete a specific grant from the role. This may be a subset or a superset of the grants the role has. In case of + a subset, the role will retain the grants not specified. If the `cascade` parameter is true, grant revocation + will have a cascading effect - that is, if a principal has specific grants on a subresource, and grants are revoked + on a parent resource, the grants present on the subresource will be revoked as well. By default, this behavior + is disabled and grant revocation only affects the specified resource. + parameters: + - name: cascade + in: query + schema: + type: boolean + default: false + description: If true, the grant revocation cascades to all subresources. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RevokeGrantRequest" + responses: + 201: + description: "Successful response" + 403: + description: "The principal is not authorized to create grants" + 404: + description: "The catalog or the role does not exist" + +components: + securitySchemes: + OAuth2: + type: oauth2 + description: Uses OAuth 2 with client credentials flow + flows: + implicit: + authorizationUrl: "{scheme}://{host}/api/v1/oauth/tokens" + scopes: {} + + schemas: + Catalogs: + type: object + description: A list of Catalog objects + properties: + catalogs: + type: array + items: + $ref: "#/components/schemas/Catalog" + required: + - catalogs + + CreateCatalogRequest: + type: object + description: Request to create a new catalog + properties: + catalog: + $ref: "#/components/schemas/Catalog" + required: + - catalog + + Catalog: + type: object + description: A catalog object. A catalog may be internal or external. External catalogs are managed entirely by + an external catalog interface. Third party catalogs may be other Iceberg REST implementations or other services + with their own proprietary APIs + properties: + type: + type: string + enum: + - INTERNAL + - EXTERNAL + description: the type of catalog - internal or external + default: INTERNAL + name: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + description: The name of the catalog + properties: + type: object + properties: + default-base-location: + type: string + additionalProperties: + type: string + required: + - default-base-location + createTimestamp: + type: integer + format: "int64" + description: The creation time represented as unix epoch timestamp in milliseconds + lastUpdateTimestamp: + type: integer + format: "int64" + description: The last update time represented as unix epoch timestamp in milliseconds + entityVersion: + type: integer + description: The version of the catalog object used to determine if the catalog metadata has changed + storageConfigInfo: + $ref: "#/components/schemas/StorageConfigInfo" + required: + - name + - type + - storageConfigInfo + - properties + discriminator: + propertyName: type + mapping: + INTERNAL: "#/components/schemas/PolarisCatalog" + EXTERNAL: "#/components/schemas/ExternalCatalog" + + + PolarisCatalog: + type: object + allOf: + - $ref: "#/components/schemas/Catalog" + description: The base catalog type - this contains all the fields necessary to construct an INTERNAL catalog + + ExternalCatalog: + description: An externally managed catalog + type: object + allOf: + - $ref: "#/components/schemas/Catalog" + - type: object + properties: + connectionConfigInfo: + $ref: "#/components/schemas/ConnectionConfigInfo" + + ConnectionConfigInfo: + type: object + description: A connection configuration representing a remote catalog service. IMPORTANT - Specifying a + ConnectionConfigInfo in an ExternalCatalog is currently an experimental API and is subject to change. + properties: + connectionType: + type: string + enum: + - ICEBERG_REST + description: The type of remote catalog service represented by this connection + uri: + type: string + description: URI to the remote catalog service + authenticationParameters: + $ref: "#/components/schemas/AuthenticationParameters" + required: + - connectionType + discriminator: + propertyName: connectionType + mapping: + ICEBERG_REST: "#/components/schemas/IcebergRestConnectionConfigInfo" + + IcebergRestConnectionConfigInfo: + type: object + description: Configuration necessary for connecting to an Iceberg REST Catalog + allOf: + - $ref: '#/components/schemas/ConnectionConfigInfo' + properties: + remoteCatalogName: + type: string + description: The name of a remote catalog instance within the remote catalog service; in some older systems + this is specified as the 'warehouse' when multiple logical catalogs are served under the same base + uri, and often translates into a 'prefix' added to all REST resource paths + + AuthenticationParameters: + type: object + description: Authentication-specific information for a REST connection + properties: + authenticationType: + type: string + enum: + - OAUTH + - BEARER + description: The type of authentication to use when connecting to the remote rest service + required: + - authenticationType + discriminator: + propertyName: authenticationType + mapping: + OAUTH: "#/components/schemas/OAuthClientCredentialsParameters" + BEARER: "#/components/schemas/BearerAuthenticationParameters" + + OAuthClientCredentialsParameters: + type: object + description: OAuth authentication based on client_id/client_secret + allOf: + - $ref: '#/components/schemas/AuthenticationParameters' + properties: + tokenUri: + type: string + description: Token server URI + clientId: + type: string + description: oauth client id + clientSecret: + type: string + format: password + description: oauth client secret (input-only) + scopes: + type: array + items: + type: string + description: oauth scopes to specify when exchanging for a short-lived access token + + BearerAuthenticationParameters: + type: object + description: Bearer authentication directly embedded in request auth headers + allOf: + - $ref: '#/components/schemas/AuthenticationParameters' + properties: + bearerToken: + type: string + format: password + description: Bearer token (input-only) + + StorageConfigInfo: + type: object + description: A storage configuration used by catalogs + properties: + storageType: + type: string + enum: + - S3 + - GCS + - AZURE + - FILE + description: The cloud provider type this storage is built on. FILE is supported for testing purposes only + allowedLocations: + type: array + items: + type: string + example: "For AWS [s3://bucketname/prefix/], for AZURE [abfss://container@storageaccount.blob.core.windows.net/prefix/], for GCP [gs://bucketname/prefix/]" + required: + - storageType + discriminator: + propertyName: storageType + mapping: + S3: "#/components/schemas/AwsStorageConfigInfo" + AZURE: "#/components/schemas/AzureStorageConfigInfo" + GCS: "#/components/schemas/GcpStorageConfigInfo" + FILE: "#/components/schemas/FileStorageConfigInfo" + + AwsStorageConfigInfo: + type: object + description: aws storage configuration info + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + roleArn: + type: string + description: the aws role arn that grants privileges on the S3 buckets + example: "arn:aws:iam::123456789001:principal/abc1-b-self1234" + externalId: + type: string + description: an optional external id used to establish a trust relationship with AWS in the trust policy + userArn: + type: string + description: the aws user arn used to assume the aws role + example: "arn:aws:iam::123456789001:user/abc1-b-self1234" + region: + type: string + description: the aws region where data is stored + example: "us-east-2" + required: + - roleArn + + AzureStorageConfigInfo: + type: object + description: azure storage configuration info + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + tenantId: + type: string + description: the tenant id that the storage accounts belong to + multiTenantAppName: + type: string + description: the name of the azure client application + consentUrl: + type: string + description: URL to the Azure permissions request page + required: + - tenantId + + GcpStorageConfigInfo: + type: object + description: gcp storage configuration info + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + properties: + gcsServiceAccount: + type: string + description: a Google cloud storage service account + + FileStorageConfigInfo: + type: object + description: file storage configuration info + allOf: + - $ref: '#/components/schemas/StorageConfigInfo' + + UpdateCatalogRequest: + description: Updates to apply to a Catalog. Any fields which are required in the Catalog + will remain unaltered if omitted from the contents of this Update request. + type: object + properties: + currentEntityVersion: + type: integer + description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version. + properties: + type: object + additionalProperties: + type: string + storageConfigInfo: + $ref: "#/components/schemas/StorageConfigInfo" + + Principals: + description: A list of Principals + type: object + properties: + principals: + type: array + items: + $ref: "#/components/schemas/Principal" + required: + - principals + + PrincipalWithCredentials: + description: A user with its client id and secret. This type is returned when a new principal is created or when its + credentials are rotated + type: object + properties: + principal: + $ref: "#/components/schemas/Principal" + credentials: + type: object + properties: + clientId: + type: string + clientSecret: + type: string + format: password + required: + - principal + - credentials + + CreatePrincipalRequest: + type: object + properties: + principal: + $ref: '#/components/schemas/Principal' + credentialRotationRequired: + type: boolean + description: If true, the initial credentials can only be used to call rotateCredentials + + Principal: + description: A Polaris principal. + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + clientId: + type: string + description: The output-only OAuth clientId associated with this principal if applicable + properties: + type: object + additionalProperties: + type: string + createTimestamp: + type: integer + format: "int64" + lastUpdateTimestamp: + type: integer + format: "int64" + entityVersion: + type: integer + description: The version of the principal object used to determine if the principal metadata has changed + required: + - name + + UpdatePrincipalRequest: + description: Updates to apply to a Principal + type: object + properties: + currentEntityVersion: + type: integer + description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version. + properties: + type: object + additionalProperties: + type: string + required: + - currentEntityVersion + - properties + + PrincipalRoles: + type: object + properties: + roles: + type: array + items: + $ref: "#/components/schemas/PrincipalRole" + required: + - roles + + GrantPrincipalRoleRequest: + type: object + properties: + principalRole: + $ref: '#/components/schemas/PrincipalRole' + + CreatePrincipalRoleRequest: + type: object + properties: + principalRole: + $ref: '#/components/schemas/PrincipalRole' + + PrincipalRole: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + description: The name of the role + properties: + type: object + additionalProperties: + type: string + createTimestamp: + type: integer + format: "int64" + lastUpdateTimestamp: + type: integer + format: "int64" + entityVersion: + type: integer + description: The version of the principal role object used to determine if the principal role metadata has changed + required: + - name + + UpdatePrincipalRoleRequest: + description: Updates to apply to a Principal Role + type: object + properties: + currentEntityVersion: + type: integer + description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version. + properties: + type: object + additionalProperties: + type: string + required: + - currentEntityVersion + - properties + + CatalogRoles: + type: object + properties: + roles: + type: array + items: + $ref: "#/components/schemas/CatalogRole" + description: The list of catalog roles + required: + - roles + + GrantCatalogRoleRequest: + type: object + properties: + catalogRole: + $ref: '#/components/schemas/CatalogRole' + + CreateCatalogRoleRequest: + type: object + properties: + catalogRole: + $ref: '#/components/schemas/CatalogRole' + + CatalogRole: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 256 + pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$' + description: The name of the role + properties: + type: object + additionalProperties: + type: string + createTimestamp: + type: integer + format: "int64" + lastUpdateTimestamp: + type: integer + format: "int64" + entityVersion: + type: integer + description: The version of the catalog role object used to determine if the catalog role metadata has changed + required: + - name + + UpdateCatalogRoleRequest: + description: Updates to apply to a Catalog Role + type: object + properties: + currentEntityVersion: + type: integer + description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version. + properties: + type: object + additionalProperties: + type: string + required: + - currentEntityVersion + - properties + + ViewPrivilege: + type: string + enum: + - CATALOG_MANAGE_ACCESS + - VIEW_DROP + - VIEW_LIST + - VIEW_READ_PROPERTIES + - VIEW_WRITE_PROPERTIES + - VIEW_FULL_METADATA + + TablePrivilege: + type: string + enum: + - CATALOG_MANAGE_ACCESS + - TABLE_DROP + - TABLE_LIST + - TABLE_READ_PROPERTIES + - TABLE_WRITE_PROPERTIES + - TABLE_READ_DATA + - TABLE_WRITE_DATA + - TABLE_FULL_METADATA + + NamespacePrivilege: + type: string + enum: + - CATALOG_MANAGE_ACCESS + - CATALOG_MANAGE_CONTENT + - CATALOG_MANAGE_METADATA + - NAMESPACE_CREATE + - TABLE_CREATE + - VIEW_CREATE + - NAMESPACE_DROP + - TABLE_DROP + - VIEW_DROP + - NAMESPACE_LIST + - TABLE_LIST + - VIEW_LIST + - NAMESPACE_READ_PROPERTIES + - TABLE_READ_PROPERTIES + - VIEW_READ_PROPERTIES + - NAMESPACE_WRITE_PROPERTIES + - TABLE_WRITE_PROPERTIES + - VIEW_WRITE_PROPERTIES + - TABLE_READ_DATA + - TABLE_WRITE_DATA + - NAMESPACE_FULL_METADATA + - TABLE_FULL_METADATA + - VIEW_FULL_METADATA + + CatalogPrivilege: + type: string + enum: + - CATALOG_MANAGE_ACCESS + - CATALOG_MANAGE_CONTENT + - CATALOG_MANAGE_METADATA + - CATALOG_READ_PROPERTIES + - CATALOG_WRITE_PROPERTIES + - NAMESPACE_CREATE + - TABLE_CREATE + - VIEW_CREATE + - NAMESPACE_DROP + - TABLE_DROP + - VIEW_DROP + - NAMESPACE_LIST + - TABLE_LIST + - VIEW_LIST + - NAMESPACE_READ_PROPERTIES + - TABLE_READ_PROPERTIES + - VIEW_READ_PROPERTIES + - NAMESPACE_WRITE_PROPERTIES + - TABLE_WRITE_PROPERTIES + - VIEW_WRITE_PROPERTIES + - TABLE_READ_DATA + - TABLE_WRITE_DATA + - NAMESPACE_FULL_METADATA + - TABLE_FULL_METADATA + - VIEW_FULL_METADATA + + AddGrantRequest: + type: object + properties: + grant: + $ref: '#/components/schemas/GrantResource' + + RevokeGrantRequest: + type: object + properties: + grant: + $ref: '#/components/schemas/GrantResource' + + ViewGrant: + allOf: + - $ref: '#/components/schemas/GrantResource' + - type: object + properties: + namespace: + type: array + items: + type: string + viewName: + type: string + minLength: 1 + maxLength: 256 + privilege: + $ref: '#/components/schemas/ViewPrivilege' + required: + - namespace + - viewName + - privilege + + TableGrant: + allOf: + - $ref: '#/components/schemas/GrantResource' + - type: object + properties: + namespace: + type: array + items: + type: string + tableName: + type: string + minLength: 1 + maxLength: 256 + privilege: + $ref: '#/components/schemas/TablePrivilege' + required: + - namespace + - tableName + - privilege + + NamespaceGrant: + allOf: + - $ref: '#/components/schemas/GrantResource' + - type: object + properties: + namespace: + type: array + items: + type: string + privilege: + $ref: '#/components/schemas/NamespacePrivilege' + required: + - namespace + - privilege + + + CatalogGrant: + allOf: + - $ref: '#/components/schemas/GrantResource' + - type: object + properties: + privilege: + $ref: '#/components/schemas/CatalogPrivilege' + required: + - privilege + + GrantResource: + type: object + discriminator: + propertyName: type + mapping: + catalog: '#/components/schemas/CatalogGrant' + namespace: '#/components/schemas/NamespaceGrant' + table: '#/components/schemas/TableGrant' + view: '#/components/schemas/ViewGrant' + properties: + type: + type: string + enum: + - catalog + - namespace + - table + - view + required: + - type + + GrantResources: + type: object + properties: + grants: + type: array + items: + $ref: "#/components/schemas/GrantResource" + required: + - grants \ No newline at end of file diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java new file mode 100644 index 00000000..f40e6b1d --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.List; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants; +import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; +import org.apache.polaris.tools.sync.polaris.planning.NoOpSyncPlanner; +import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class AccessControlAwarePlannerTest { + + private static final PrincipalRole omnipotentPrincipalRoleSource = + new PrincipalRole() + .name("omnipotent-principal-XXXXX") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + private static final PrincipalRole omnipotentPrincipalRoleTarget = + new PrincipalRole() + .name("omnipotent-principal-YYYYY") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + @Test + public void filtersOmnipotentPrincipalRoles() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planPrincipalRoleSync( + List.of(omnipotentPrincipalRoleSource), List.of(omnipotentPrincipalRoleTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget)); + } + + private static final PrincipalRole serviceAdminSource = new PrincipalRole().name("service_admin"); + + private static final PrincipalRole serviceAdminTarget = new PrincipalRole().name("service_admin"); + + @Test + public void filtersServiceAdmin() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planPrincipalRoleSync( + List.of(serviceAdminSource), List.of(serviceAdminTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget)); + } + + private static final CatalogRole omnipotentCatalogRoleSource = + new CatalogRole() + .name("omnipotent-principal-XXXXX") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + private static final CatalogRole omnipotentCatalogRoleTarget = + new CatalogRole() + .name("omnipotent-principal-YYYYY") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + @Test + public void filtersOmnipotentCatalogRolesAndChildren() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planCatalogRoleSync( + "catalogName", + List.of(omnipotentCatalogRoleSource), + List.of(omnipotentCatalogRoleTarget)); + + Assertions.assertTrue( + plan.entitiesToSkipAndSkipChildren().contains(omnipotentCatalogRoleSource)); + Assertions.assertTrue( + plan.entitiesToSkipAndSkipChildren().contains(omnipotentCatalogRoleTarget)); + } + + private static final CatalogRole catalogAdminSource = new CatalogRole().name("catalog_admin"); + + private static final CatalogRole catalogAdminTarget = new CatalogRole().name("catalog_admin"); + + @Test + public void filtersCatalogAdminAndChildren() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planCatalogRoleSync( + "catalogName", List.of(catalogAdminSource), List.of(catalogAdminTarget)); + + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(catalogAdminSource)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(catalogAdminTarget)); + } + + @Test + public void filtersOutAssignmentOfOmnipotentPrincipalRoles() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planAssignPrincipalRolesToCatalogRolesSync( + "catalogName", + "catalogRoleName", + List.of(omnipotentPrincipalRoleSource), + List.of(omnipotentPrincipalRoleTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget)); + } + + @Test + public void filtersOutAssignmentOfServiceAdmin() { + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + accessControlAwarePlanner.planAssignPrincipalRolesToCatalogRolesSync( + "catalogName", + "catalogRoleName", + List.of(serviceAdminSource), + List.of(serviceAdminTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget)); + } +} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java new file mode 100644 index 00000000..5d875017 --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.List; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; +import org.apache.polaris.tools.sync.polaris.planning.NoOpSyncPlanner; +import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ModificationAwarePlannerTest { + + private static final PrincipalRole principalRole = new PrincipalRole().name("principal-role"); + + private static final PrincipalRole modifiedPrincipalRole = + new PrincipalRole().name("principal-role").putPropertiesItem("newproperty", "newvalue"); + + @Test + public void testPrincipalRoleNotModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planPrincipalRoleSync(List.of(principalRole), List.of(principalRole)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(principalRole)); + } + + @Test + public void testPrincipalRoleModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planPrincipalRoleSync( + List.of(principalRole), List.of(modifiedPrincipalRole)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(principalRole)); + } + + private static final CatalogRole catalogRole = new CatalogRole().name("catalog-role"); + + private static final CatalogRole modifiedCatalogRole = + new CatalogRole().name("catalog-role").putPropertiesItem("newproperty", "newvalue"); + + @Test + public void testCatalogRoleNotModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogRoleSync( + "catalog", List.of(catalogRole), List.of(catalogRole)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(catalogRole)); + } + + @Test + public void testCatalogRoleModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogRoleSync( + "catalog", List.of(catalogRole), List.of(modifiedCatalogRole)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(catalogRole)); + } + + private static final GrantResource grant = + new GrantResource().type(GrantResource.TypeEnum.CATALOG); + + @Test + public void testGrantNotRevoked() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planGrantSync("catalog", "catalogRole", List.of(grant), List.of(grant)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(grant)); + } + + private static final Catalog catalog = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new AwsStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.S3) + .roleArn("roleArn") + .userArn("userArn") + .externalId("externalId") + .region("region")); + + private static final Catalog catalogWithTypeChange = + new ExternalCatalog() + .name("catalog") + .type(Catalog.TypeEnum.EXTERNAL) // changed type + .properties(new CatalogProperties()) + .storageConfigInfo( + new AwsStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.S3) + .roleArn("roleArn") + .userArn("userArn") + .externalId("externalId") + .region("region")); + + private static final Catalog catalogWithStorageConfigInfoChange = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.EXTERNAL) // changed type + .properties(new CatalogProperties()) + .storageConfigInfo( + new AzureStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .consentUrl("consentUrl") + .tenantId("tenantId") + .multiTenantAppName("multiTenantAppName")); + + private static final Catalog catalogWithOnlyUserArnChange = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new AwsStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.S3) + .roleArn("roleArn") + .userArn("userArnChanged") // only user arn changed + .externalId("externalId") + .region("region")); + + private static final Catalog catalogWithPropertyChange = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties().putAdditionalProperty("newproperty", "newvalue")) + .storageConfigInfo( + new AwsStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.S3) + .roleArn("roleArn") + .userArn("userArn") + .externalId("externalId") + .region("region")); + + private static final Catalog azureCatalog = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new AzureStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .consentUrl("consentUrl") + .multiTenantAppName("multiTenantAppName") + .tenantId("tenantId")); + + private static final Catalog azureCatalogConsentUrlAndTenantAppNameChange = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new AzureStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .consentUrl("consentUrlChanged") + .multiTenantAppName("multiTenantAppNameChanged") + .tenantId("tenantId")); + + private static final Catalog gcpCatalog = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new GcpStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .gcsServiceAccount("gcsServiceAccount")); + + private static final Catalog gcpCatalogGcsServiceAccountChange = + new PolarisCatalog() + .name("catalog") + .type(Catalog.TypeEnum.INTERNAL) + .properties(new CatalogProperties()) + .storageConfigInfo( + new GcpStorageConfigInfo() + .storageType(StorageConfigInfo.StorageTypeEnum.AZURE) + .gcsServiceAccount("gcsServiceAccountChanged")); + + @Test + public void testCatalogNotModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync(List.of(catalog), List.of(catalog)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(catalog)); + } + + @Test + public void testCatalogTypeModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync(List.of(catalogWithTypeChange), List.of(catalog)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithTypeChange)); + } + + @Test + public void testCatalogStorageConfigInfoModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync( + List.of(catalogWithStorageConfigInfoChange), List.of(catalog)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithStorageConfigInfoChange)); + } + + @Test + public void testCatalogPropertiesModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync(List.of(catalogWithPropertyChange), List.of(catalog)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithPropertyChange)); + } + + @Test + public void testOnlyUserArnModifiedForAws() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync( + List.of(catalogWithOnlyUserArnChange), List.of(catalog)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(catalogWithOnlyUserArnChange)); + } + + @Test + public void testOnlyConsentUrlAndTenantAppNameChangeAzure() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync( + List.of(azureCatalogConsentUrlAndTenantAppNameChange), List.of(azureCatalog)); + + Assertions.assertTrue( + plan.entitiesNotModified().contains(azureCatalogConsentUrlAndTenantAppNameChange)); + } + + @Test + public void testOnlyGcsServiceAccountChangeGCP() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planCatalogSync( + List.of(gcpCatalogGcsServiceAccountChange), List.of(gcpCatalog)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(gcpCatalogGcsServiceAccountChange)); + } +} diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java new file mode 100644 index 00000000..c9a8480f --- /dev/null +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.List; +import java.util.Set; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SourceParitySynchronizationPlannerTest { + + private static final Catalog CATALOG_1 = new Catalog().name("catalog-1"); + + private static final Catalog CATALOG_2 = new Catalog().name("catalog-2"); + + private static final Catalog CATALOG_3 = new Catalog().name("catalog-3"); + + @Test + public void testCreatesNewCatalogOverwritesOldCatalogRemovesDroppedCatalog() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planCatalogSync(List.of(CATALOG_1, CATALOG_2), List.of(CATALOG_2, CATALOG_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_3)); + } + + private static final PrincipalRole PRINCIPAL_ROLE_1 = + new PrincipalRole().name("principal-role-1"); + + private static final PrincipalRole PRINCIPAL_ROLE_2 = + new PrincipalRole().name("principal-role-2"); + + private static final PrincipalRole PRINCIPAL_ROLE_3 = + new PrincipalRole().name("principal-role-3"); + + @Test + public void testCreatesNewPrincipalRoleOverwritesOldPrincipalRoleRemovesDroppedPrincipalRole() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planPrincipalRoleSync( + List.of(PRINCIPAL_ROLE_1, PRINCIPAL_ROLE_2), + List.of(PRINCIPAL_ROLE_2, PRINCIPAL_ROLE_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_3)); + } + + private static final CatalogRole CATALOG_ROLE_1 = new CatalogRole().name("catalog-role-1"); + + private static final CatalogRole CATALOG_ROLE_2 = new CatalogRole().name("catalog-role-2"); + + private static final CatalogRole CATALOG_ROLE_3 = new CatalogRole().name("catalog-role-3"); + + @Test + public void testCreatesNewCatalogRoleOverwritesOldCatalogRoleRemovesDroppedCatalogRole() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planCatalogRoleSync( + "catalog", + List.of(CATALOG_ROLE_1, CATALOG_ROLE_2), + List.of(CATALOG_ROLE_2, CATALOG_ROLE_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_ROLE_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_ROLE_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_ROLE_3)); + } + + private static final GrantResource GRANT_1 = + new GrantResource().type(GrantResource.TypeEnum.CATALOG); + + private static final GrantResource GRANT_2 = + new GrantResource().type(GrantResource.TypeEnum.NAMESPACE); + + private static final GrantResource GRANT_3 = + new GrantResource().type(GrantResource.TypeEnum.TABLE); + + @Test + public void testCreatesNewGrantResourceRemovesDroppedGrantResource() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planGrantSync( + "catalog", "catalogRole", List.of(GRANT_1, GRANT_2), List.of(GRANT_2, GRANT_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(GRANT_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_1)); + + // special case: no concept of overwriting a grant + Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_2)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(GRANT_3)); + } + + private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_1 = + new PrincipalRole().name("principal-role-1"); + + private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_2 = + new PrincipalRole().name("principal-role-2"); + + private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_3 = + new PrincipalRole().name("principal-role-3"); + + @Test + public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRole() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planAssignPrincipalRolesToCatalogRolesSync( + "catalog", + "catalogRole", + List.of(ASSIGNED_PRINCIPAL_ROLE_1, ASSIGNED_PRINCIPAL_ROLE_2), + List.of(ASSIGNED_PRINCIPAL_ROLE_2, ASSIGNED_PRINCIPAL_ROLE_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_1)); + + // special case: no concept of overwriting the assignment of a principal role + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_2)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_3)); + } + + private static final Namespace NS_1 = Namespace.of("ns1"); + + private static final Namespace NS_2 = Namespace.of("ns2"); + + private static final Namespace NS_3 = Namespace.of("ns3"); + + @Test + public void testCreatesNewNamespaceOverwritesOldNamespaceDropsDroppedNamespace() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + SynchronizationPlan plan = + planner.planNamespaceSync( + "catalog", Namespace.empty(), List.of(NS_1, NS_2), List.of(NS_2, NS_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(NS_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(NS_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(NS_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(NS_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(NS_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(NS_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(NS_3)); + } + + private static final TableIdentifier TABLE_1 = TableIdentifier.of("ns", "table1"); + + private static final TableIdentifier TABLE_2 = TableIdentifier.of("ns", "table2"); + + private static final TableIdentifier TABLE_3 = TableIdentifier.of("ns", "table3"); + + @Test + public void + testCreatesNewTableIdentifierOverwritesOldTableIdentifierRevokesDroppedTableIdentifier() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planTableSync( + "catalog", Namespace.empty(), Set.of(TABLE_1, TABLE_2), Set.of(TABLE_2, TABLE_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(TABLE_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(TABLE_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(TABLE_3)); + } +} diff --git a/polaris-synchronizer/build.gradle.kts b/polaris-synchronizer/build.gradle.kts new file mode 100644 index 00000000..a0f2bee2 --- /dev/null +++ b/polaris-synchronizer/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + id("java") +} + +group = "org.apache.polaris.tools" +version = "0.1-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.10.0")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +// don't build jar in the root project +tasks.named("jar").configure { + enabled = false +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/polaris-synchronizer/cli/build.gradle.kts b/polaris-synchronizer/cli/build.gradle.kts new file mode 100644 index 00000000..e637d630 --- /dev/null +++ b/polaris-synchronizer/cli/build.gradle.kts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `java-library` + `maven-publish` + signing + id("com.github.johnrengelman.shadow") version "8.1.1" +} + +repositories { + mavenCentral() +} + +dependencies { + implementation(project(":polaris-synchronizer-api")) + + implementation("info.picocli:picocli:4.7.6") + implementation("org.slf4j:log4j-over-slf4j:2.0.17") + implementation("org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.7.1") + implementation("org.apache.commons:commons-csv:1.13.0") + runtimeOnly("ch.qos.logback:logback-classic:1.5.17") + + testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.0") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0") + testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.10.0") +} + +tasks.withType().configureEach { useJUnitPlatform() } + +val mainClassName = "org.apache.polaris.tools.sync.polaris.PolarisSynchronizerCLI" + +tasks.named("shadowJar") { + manifest { + attributes["Main-Class"] = mainClassName + } + outputs.cacheIf { false } + archiveClassifier.set("") // no `-all` + archiveVersion.set("") // optional + isZip64 = true +} + +tasks.named("build") { + dependsOn("shadowJar") +} + +// Optionally disable raw JAR +tasks.named("jar") { + enabled = false +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java new file mode 100644 index 00000000..596dc4e7 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.access.AccessControlService; +import org.apache.polaris.tools.sync.polaris.options.PolarisOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +@CommandLine.Command( + name = "create-omnipotent-principal", + mixinStandardHelpOptions = true, + sortOptions = false, + description = + "Creates a principal, associated principal role, and associated catalog role for each catalog " + + "with appropriate access permissions.") +public class CreateOmnipotentPrincipalCommand implements Callable { + + private final Logger consoleLog = LoggerFactory.getLogger("console-log"); + + @CommandLine.ArgGroup(exclusive = false, multiplicity = "1", heading = "Polaris options: %n") + private PolarisOptions options; + + @CommandLine.Option( + names = {"--replace"}, + description = { + "Optional flag to enable overwriting the existing omnipotent principal and associated entity if it exists." + }) + private boolean replace; + + @CommandLine.Option( + names = {"--write-access"}, + description = { + "Optional flag to create the principal with write access to every catalog. This is required if " + + "the Polaris instance is the target of a sync." + }) + private boolean withWriteAccess; + + @CommandLine.Option( + names = {"--concurrency"}, + defaultValue = "1", + description = { + "Optional flag to specify the number of concurrent threads to use to setup catalog roles." + }) + private int concurrency; + + @Override + public Integer call() throws Exception { + PolarisService polaris = options.buildService(); + AccessControlService accessControlService = new AccessControlService(polaris); + + PrincipalWithCredentials principalWithCredentials; + + try { + principalWithCredentials = accessControlService.createOmnipotentPrincipal(replace); + } catch (Exception e) { + consoleLog.error("Failed to create omnipotent principal.", e); + return 1; + } + + consoleLog.info( + "Created omnipotent principal {}.", principalWithCredentials.getPrincipal().getName()); + + PrincipalRole principalRole; + + try { + principalRole = + accessControlService.createAndAssignPrincipalRole(principalWithCredentials, replace); + } catch (Exception e) { + consoleLog.error("Failed to create omnipotent principal role and assign it to principal.", e); + return 1; + } + + consoleLog.info( + "Created omnipotent principal role {} and assigned it to omnipotent principal {}.", + principalWithCredentials.getPrincipal().getName(), + principalRole.getName()); + + List catalogs = polaris.listCatalogs(); + + consoleLog.info("Identified {} catalogs to create catalog roles for.", catalogs.size()); + + final String permissionLevel = withWriteAccess ? "write" : "readonly"; + + AtomicInteger completedCatalogSetups = new AtomicInteger(0); + + Queue failedCatalogs = new ConcurrentLinkedQueue<>(); + + ExecutorService executor = Executors.newFixedThreadPool(concurrency); + + List> futures = new ArrayList<>(); + + for (Catalog catalog : catalogs) { + CompletableFuture future = + CompletableFuture.runAsync( + () -> { + try { + accessControlService.setupOmnipotentRoleForCatalog( + catalog.getName(), principalRole, replace, withWriteAccess); + } catch (Exception e) { + failedCatalogs.add(catalog); + consoleLog.error( + "Failed to setup omnipotent catalog role for catalog {} with {} access. - {}/{}", + catalog.getName(), + permissionLevel, + completedCatalogSetups.getAndIncrement(), + catalogs.size(), + e); + } + + consoleLog.info( + "Finished omnipotent principal setup for catalog {} with {} access. - {}/{}", + catalog.getName(), + permissionLevel, + completedCatalogSetups.incrementAndGet(), + catalogs.size()); + }, + executor); + + futures.add(future); + } + + futures.forEach(CompletableFuture::join); + + consoleLog.info( + "Encountered issues creating catalog roles for the following catalogs: {}", + failedCatalogs.stream().map(Catalog::getName).toList()); + + consoleLog.info( + "\n======================================================\n" + + "Omnipotent Principal Credentials:\n" + + "\tname = {}\n" + + "\tclientId = {}\n" + + "\tclientSecret = {}\n" + + "======================================================", + principalWithCredentials.getPrincipal().getName(), + principalWithCredentials.getCredentials().getClientId(), + principalWithCredentials.getCredentials().getClientSecret()); + + return 0; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java new file mode 100644 index 00000000..e391214c --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.tools.sync.polaris.catalog.ETagService; + +/** Implementation that stores/loads ETags to/from a CSV file. */ +public class CsvETagService implements ETagService, Closeable { + + private static final String CATALOG_HEADER = "Catalog"; + + private static final String TABLE_ID_HEADER = "TableIdentifier"; + + private static final String ETAG_HEADER = "ETag"; + + private static final String[] HEADERS = {CATALOG_HEADER, TABLE_ID_HEADER, ETAG_HEADER}; + + private final File file; + + private final Map> tablesByCatalogName; + + public CsvETagService(File file) throws IOException { + this.tablesByCatalogName = new HashMap<>(); + this.file = file; + + if (file.exists()) { + CSVFormat readerCSVFormat = + CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get(); + + CSVParser parser = + CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat); + + for (CSVRecord record : parser.getRecords()) { + this.tablesByCatalogName.putIfAbsent(record.get(CATALOG_HEADER), new HashMap<>()); + + TableIdentifier tableId = TableIdentifier.parse(record.get(TABLE_ID_HEADER)); + + this.tablesByCatalogName + .get(record.get(CATALOG_HEADER)) + .put(tableId, record.get(ETAG_HEADER)); + } + + parser.close(); + } + } + + @Override + public String getETag(String catalogName, TableIdentifier tableIdentifier) { + if (tablesByCatalogName.get(catalogName) != null) { + return tablesByCatalogName + .get(catalogName) + .get(tableIdentifier); // will return null anyway if table id not available + } + return null; + } + + @Override + public void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag) { + this.tablesByCatalogName.putIfAbsent(catalogName, new HashMap<>()); + this.tablesByCatalogName.get(catalogName).put(tableIdentifier, etag); + } + + @Override + public void close() throws IOException { + BufferedWriter writer = Files.newBufferedWriter(file.toPath(), UTF_8); + + writer.write(""); // clear file + + CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS).get(); + + CSVPrinter printer = new CSVPrinter(writer, csvFormat); + + // write etags to file + tablesByCatalogName.forEach( + (catalogName, etagsByTable) -> { + etagsByTable.forEach( + (tableIdentifier, etag) -> { + try { + printer.printRecord(catalogName, tableIdentifier.toString(), etag); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + }); + + printer.flush(); + printer.close(); + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java new file mode 100644 index 00000000..08c36fc7 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import picocli.CommandLine; + +@CommandLine.Command( + name = "polaris-synchronizer", + mixinStandardHelpOptions = true, + subcommands = {SyncPolarisCommand.class, CreateOmnipotentPrincipalCommand.class}) +public class PolarisSynchronizerCLI { + + public PolarisSynchronizerCLI() {} + + public static void main(String[] args) { + CommandLine commandLine = + new CommandLine(new PolarisSynchronizerCLI()) + .setExecutionExceptionHandler( + (ex, cmd, parseResult) -> { + cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex)); + return 1; + }); + commandLine.setUsageHelpWidth(150); + int exitCode = commandLine.execute(args); + System.exit(exitCode); + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java new file mode 100644 index 00000000..eda18c58 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.concurrent.Callable; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.catalog.ETagService; +import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagService; +import org.apache.polaris.tools.sync.polaris.options.SourceOmniPotentPrincipalOptions; +import org.apache.polaris.tools.sync.polaris.options.SourcePolarisOptions; +import org.apache.polaris.tools.sync.polaris.options.TargetOmnipotentPrincipal; +import org.apache.polaris.tools.sync.polaris.options.TargetPolarisOptions; +import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; +import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; +import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +@CommandLine.Command( + name = "sync-polaris", + mixinStandardHelpOptions = true, + sortOptions = false, + description = + "Idempotent synchronization of one Polaris instance to another. Entities will not be removed from the source Polaris instance.") +public class SyncPolarisCommand implements Callable { + + private final Logger consoleLog = LoggerFactory.getLogger("console-log"); + + @CommandLine.ArgGroup( + exclusive = false, + multiplicity = "1", + heading = "Source Polaris options: %n") + private SourcePolarisOptions sourcePolarisOptions; + + @CommandLine.ArgGroup( + exclusive = false, + multiplicity = "1", + heading = "Target Polaris options: %n") + private TargetPolarisOptions targetPolarisOptions; + + @CommandLine.ArgGroup( + exclusive = false, + multiplicity = "1", + heading = "Source Polaris Omnipotent Principal Options: %n") + private SourceOmniPotentPrincipalOptions sourceOmniPotentPrincipalOptions; + + @CommandLine.ArgGroup( + exclusive = false, + multiplicity = "1", + heading = "Target Polaris Omnipotent Principal Options: %n") + private TargetOmnipotentPrincipal targetOmniPotentPrincipalOptions; + + @CommandLine.Option( + names = {"--etag-file"}, + description = "The file path of the file to retrieve and store table ETags from.") + private String etagFilePath; + + @Override + public Integer call() throws Exception { + SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner(); + SynchronizationPlanner modificationAwareSourceParityPlanner = + new ModificationAwarePlanner(sourceParityPlanner); + SynchronizationPlanner accessControlAwarePlanner = + new AccessControlAwarePlanner(modificationAwareSourceParityPlanner); + + PolarisService source = sourcePolarisOptions.buildService(); + PolarisService target = targetPolarisOptions.buildService(); + + PrincipalWithCredentials sourceOmnipotentPrincipal = + sourceOmniPotentPrincipalOptions.buildPrincipalWithCredentials(); + PrincipalWithCredentials targetOmniPotentPrincipal = + targetOmniPotentPrincipalOptions.buildPrincipalWithCredentials(); + + ETagService etagService; + + if (etagFilePath != null) { + File etagFile = new File(etagFilePath); + etagService = new CsvETagService(etagFile); + } else { + etagService = new NoOpETagService(); + } + + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (etagService instanceof Closeable closableETagService) { + try { + closableETagService.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + })); + + PolarisSynchronizer synchronizer = + new PolarisSynchronizer( + consoleLog, + accessControlAwarePlanner, + sourceOmnipotentPrincipal, + targetOmniPotentPrincipal, + source, + target, + etagService); + + synchronizer.syncPrincipalRoles(); + synchronizer.syncCatalogs(); + + return 0; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java new file mode 100644 index 00000000..c0107c16 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; + +public abstract class BaseOmnipotentPrincipalOptions { + + protected static final String PRINCIPAL_NAME = "omni-principal-name"; + + protected static final String CLIENT_ID = "omni-client-id"; + + protected static final String CLIENT_SECRET = "omni-client-secret"; + + protected String principalName; + + protected String clientId; + + protected String clientSecret; + + public abstract void setPrincipalName(String principalName); + + public abstract void setClientId(String clientId); + + public abstract void setClientSecret(String clientSecret); + + public PrincipalWithCredentials buildPrincipalWithCredentials() { + return new PrincipalWithCredentials() + .principal(new Principal().name(principalName)) + .credentials( + new PrincipalWithCredentialsCredentials() + .clientId(clientId) + .clientSecret(clientSecret)); + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java new file mode 100644 index 00000000..118489f7 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import java.io.IOException; +import org.apache.polaris.tools.sync.polaris.PolarisService; +import org.apache.polaris.tools.sync.polaris.PolarisServiceFactory; + +public abstract class BasePolarisOptions { + + protected static final String BASE_URL = "base-url"; + + protected static final String CLIENT_ID = "client-id"; + + protected static final String CLIENT_SECRET = "client-secret"; + + protected static final String SCOPE = "scope"; + + protected static final String OAUTH2_SERVER_URI = "oauth2-server-uri"; + + protected static final String ACCESS_TOKEN = "access-token"; + + protected String baseUrl; + + protected String oauth2ServerUri; + + protected String clientId; + + protected String clientSecret; + + protected String scope; + + protected String accessToken; + + public abstract String getServiceName(); + + public abstract void setBaseUrl(String baseUrl); + + public abstract void setOauth2ServerUri(String oauth2ServerUri); + + public abstract void setClientId(String clientId); + + public abstract void setClientSecret(String clientSecret); + + public abstract void setScope(String scope); + + public abstract void setAccessToken(String accessToken); + + public PolarisService buildService() throws IOException { + if (accessToken != null) { + return PolarisServiceFactory.newPolarisService(baseUrl, accessToken); + } + return PolarisServiceFactory.newPolarisService( + baseUrl, oauth2ServerUri, clientId, clientSecret, scope); + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java new file mode 100644 index 00000000..6a10e4db --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import picocli.CommandLine; + +public class PolarisOptions extends BasePolarisOptions { + + @Override + public String getServiceName() { + return "polaris"; + } + + @CommandLine.Option( + names = "--polaris-" + BASE_URL, + required = true, + description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") + @Override + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + @CommandLine.Option( + names = "--polaris-" + OAUTH2_SERVER_URI, + description = { + "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " + + "obtain an access token for the Polaris instance." + }) + @Override + public void setOauth2ServerUri(String oauth2ServerUri) { + this.oauth2ServerUri = oauth2ServerUri; + } + + @CommandLine.Option( + names = "--polaris-" + CLIENT_ID, + description = { + "(Note: required if access-token not provided) The client id for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientId(String clientId) { + this.clientId = clientId; + } + + @CommandLine.Option( + names = "--polaris-" + CLIENT_SECRET, + description = { + "(Note: required if access-token not provided) The client secret for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + } + + @CommandLine.Option( + names = "--polaris-" + SCOPE, + description = { + "(Note: required if access-token not provided) The scope that the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setScope(String scope) { + this.scope = scope; + } + + @CommandLine.Option( + names = "--polaris-" + ACCESS_TOKEN, + description = "The access token to authenticate to the Polaris instance") + @Override + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java new file mode 100644 index 00000000..90b7b46e --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import picocli.CommandLine; + +public class SourceOmniPotentPrincipalOptions extends BaseOmnipotentPrincipalOptions { + + @CommandLine.Option( + names = "--source-" + PRINCIPAL_NAME, + required = true, + description = "The principal name of the source omnipotent principal.") + @Override + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + @CommandLine.Option( + names = "--source-" + CLIENT_ID, + required = true, + description = "The client id of the source omnipotent principal.") + @Override + public void setClientId(String clientId) { + this.clientId = clientId; + } + + @CommandLine.Option( + names = "--source-" + CLIENT_SECRET, + required = true, + description = "The client secret of the source omnipotent principal.") + @Override + public void setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java new file mode 100644 index 00000000..7a811bc7 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import picocli.CommandLine; + +public class SourcePolarisOptions extends BasePolarisOptions { + + @Override + public String getServiceName() { + return "source"; + } + + @CommandLine.Option( + names = "--source-" + BASE_URL, + required = true, + description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") + @Override + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + @CommandLine.Option( + names = "--source-" + OAUTH2_SERVER_URI, + description = { + "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " + + "obtain an access token for the Polaris instance." + }) + @Override + public void setOauth2ServerUri(String oauth2ServerUri) { + this.oauth2ServerUri = oauth2ServerUri; + } + + @CommandLine.Option( + names = "--source-" + CLIENT_ID, + description = { + "(Note: required if access-token not provided) The client id for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientId(String clientId) { + this.clientId = clientId; + } + + @CommandLine.Option( + names = "--source-" + CLIENT_SECRET, + description = { + "(Note: required if access-token not provided) The client secret for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + } + + @CommandLine.Option( + names = "--source-" + SCOPE, + description = { + "(Note: required if access-token not provided) The scope that the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setScope(String scope) { + this.scope = scope; + } + + @CommandLine.Option( + names = "--source-" + ACCESS_TOKEN, + description = "The access token to authenticate to the Polaris instance") + @Override + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java new file mode 100644 index 00000000..ac03b616 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import picocli.CommandLine; + +public class TargetOmnipotentPrincipal extends BaseOmnipotentPrincipalOptions { + + @CommandLine.Option( + names = "--target-" + PRINCIPAL_NAME, + required = true, + description = "The principal name of the source omnipotent principal.") + @Override + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + @CommandLine.Option( + names = "--target-" + CLIENT_ID, + required = true, + description = "The client id of the source omnipotent principal.") + @Override + public void setClientId(String clientId) { + this.clientId = clientId; + } + + @CommandLine.Option( + names = "--target-" + CLIENT_SECRET, + required = true, + description = "The client secret of the source omnipotent principal.") + @Override + public void setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + } +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java new file mode 100644 index 00000000..6d6b2042 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.tools.sync.polaris.options; + +import picocli.CommandLine; + +public class TargetPolarisOptions extends BasePolarisOptions { + + @Override + public String getServiceName() { + return "target"; + } + + @CommandLine.Option( + names = "--target-" + BASE_URL, + required = true, + description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") + @Override + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + @CommandLine.Option( + names = "--target-" + OAUTH2_SERVER_URI, + description = { + "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " + + "obtain an access token for the Polaris instance." + }) + @Override + public void setOauth2ServerUri(String oauth2ServerUri) { + this.oauth2ServerUri = oauth2ServerUri; + } + + @CommandLine.Option( + names = "--target-" + CLIENT_ID, + description = { + "(Note: required if access-token not provided) The client id for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientId(String clientId) { + this.clientId = clientId; + } + + @CommandLine.Option( + names = "--target-" + CLIENT_SECRET, + description = { + "(Note: required if access-token not provided) The client secret for the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + } + + @CommandLine.Option( + names = "--target-" + SCOPE, + description = { + "(Note: required if access-token not provided) The scope that the principal the tool will assume" + + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." + }) + @Override + public void setScope(String scope) { + this.scope = scope; + } + + @CommandLine.Option( + names = "--target-" + ACCESS_TOKEN, + description = "The access token to authenticate to the Polaris instance") + @Override + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } +} diff --git a/polaris-synchronizer/cli/src/main/resources/logback.xml b/polaris-synchronizer/cli/src/main/resources/logback.xml new file mode 100644 index 00000000..663e03f3 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + true + + %highlight(%-5level) - %msg%n + + + + + + + + + + diff --git a/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties b/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..9355b415 --- /dev/null +++ b/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/polaris-synchronizer/gradlew b/polaris-synchronizer/gradlew new file mode 100755 index 00000000..f5feea6d --- /dev/null +++ b/polaris-synchronizer/gradlew @@ -0,0 +1,252 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/polaris-synchronizer/gradlew.bat b/polaris-synchronizer/gradlew.bat new file mode 100644 index 00000000..9b42019c --- /dev/null +++ b/polaris-synchronizer/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/polaris-synchronizer/settings.gradle.kts b/polaris-synchronizer/settings.gradle.kts new file mode 100644 index 00000000..2521c973 --- /dev/null +++ b/polaris-synchronizer/settings.gradle.kts @@ -0,0 +1,10 @@ +rootProject.name = "polaris-synchronizer" + +fun polarisSynchronizerProject(name: String) { + include("polaris-synchronizer-$name") + project(":polaris-synchronizer-$name").projectDir = file(name) +} + +polarisSynchronizerProject("api") + +polarisSynchronizerProject("cli") \ No newline at end of file From b03958e1c358eeb1bec6aaf56cbb712813b3877d Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Fri, 11 Apr 2025 16:03:56 -0700 Subject: [PATCH 02/18] Added optional principal migrations --- .../sync/polaris/PolarisSynchronizer.java | 86 +++++++++++++++++- .../access/NoOpPrincipalCredentialCaptor.java | 15 +++ .../access/PrincipalCredentialCaptor.java | 17 ++++ .../planning/AccessControlAwarePlanner.java | 48 ++++++++++ .../planning/ModificationAwarePlanner.java | 49 ++++++++++ .../polaris/planning/NoOpSyncPlanner.java | 7 ++ .../SourceParitySynchronizationPlanner.java | 29 ++++++ .../planning/SynchronizationPlanner.java | 4 + polaris-synchronizer/cli/build.gradle.kts | 4 + .../CsvPrincipalCredentialsCaptor.java | 91 +++++++++++++++++++ .../sync/polaris/SyncPolarisCommand.java | 49 +++++++++- 11 files changed, 397 insertions(+), 2 deletions(-) create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index b5954302..ca081ecb 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -30,9 +30,11 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.access.AccessControlService; +import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag; import org.apache.polaris.tools.sync.polaris.catalog.ETagService; import org.apache.polaris.tools.sync.polaris.catalog.NotModifiedException; @@ -68,6 +70,8 @@ public class PolarisSynchronizer { private final AccessControlService targetAccessControlService; + private final PrincipalCredentialCaptor principalCredentialCaptor; + private final ETagService etagService; public PolarisSynchronizer( @@ -77,7 +81,8 @@ public PolarisSynchronizer( PrincipalWithCredentials targetOmnipotentPrincipal, PolarisService source, PolarisService target, - ETagService etagService) { + ETagService etagService, + PrincipalCredentialCaptor principalCredentialCaptor) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.syncPlanner = synchronizationPlanner; @@ -95,6 +100,7 @@ public PolarisSynchronizer( targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal( targetOmnipotentPrincipal.getPrincipal().getName()); this.etagService = etagService; + this.principalCredentialCaptor = principalCredentialCaptor; } /** @@ -109,6 +115,84 @@ private int totalSyncsToComplete(SynchronizationPlan plan) { + plan.entitiesToRemove().size(); } + /** Sync principals from source to target. */ + public void syncPrincipals() { + List principalsSource; + + try { + principalsSource = source.listPrincipals(); + clientLogger.info("Listed {} principals from source.", principalsSource.size()); + } catch (Exception e) { + clientLogger.info("Failed to list principals from source.", e); + return; + } + + List principalsTarget; + + try { + principalsTarget = target.listPrincipals(); + clientLogger.info("Listed {} principals from target.", principalsTarget.size()); + } catch (Exception e) { + clientLogger.info("Failed to list principals from target.", e); + return; + } + + SynchronizationPlan principalSyncPlan = + syncPlanner.planPrincipalSync(principalsSource, principalsTarget); + + principalSyncPlan + .entitiesToSkip() + .forEach( + principal -> + clientLogger.info("Skipping principal {}.", principal.getName())); + + principalSyncPlan + .entitiesNotModified() + .forEach( + principal -> + clientLogger.info( + "No change detected for principal {}, skipping.", + principal.getName())); + + int syncsCompleted = 0; + final int totalSyncsToComplete = totalSyncsToComplete(principalSyncPlan); + + for (Principal principal : principalSyncPlan.entitiesToCreate()) { + try { + PrincipalWithCredentials createdPrincipal = target.createPrincipal(principal, false /* overwrite */); + principalCredentialCaptor.storePrincipal(createdPrincipal); + clientLogger.info("Created principal {} on target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to create principal {} on target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); + } + } + + for (Principal principal : principalSyncPlan.entitiesToOverwrite()) { + try { + PrincipalWithCredentials overwrittenPrincipal = target.createPrincipal(principal, true /* overwrite */); + principalCredentialCaptor.storePrincipal(overwrittenPrincipal); + clientLogger.info("Overwrote principal {} on target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to overwrite principal {} on target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); + } + } + + for (Principal principal : principalSyncPlan.entitiesToRemove()) { + try { + target.removePrincipal(principal.getName()); + clientLogger.info("Removed principal {} on target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to remove principal {} ont target. - {}/{}", + principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); + } + } + } + /** Sync principal roles from source to target. */ public void syncPrincipalRoles() { List principalRolesSource; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java new file mode 100644 index 00000000..e589404f --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java @@ -0,0 +1,15 @@ +package org.apache.polaris.tools.sync.polaris.access; + +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +/** + * No-op implementation that does nothing with the principal credentials. + */ +public class NoOpPrincipalCredentialCaptor implements PrincipalCredentialCaptor { + + @Override + public void storePrincipal(PrincipalWithCredentials principal) { + // do nothing + } + +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java new file mode 100644 index 00000000..44b6741c --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java @@ -0,0 +1,17 @@ +package org.apache.polaris.tools.sync.polaris.access; + +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +/** + * Captures principal credentials at the time of a principal's + * creation/overwrite on the target Polaris instance. + */ +public interface PrincipalCredentialCaptor { + + /** + * Store the principal with associated credentials at the time of its creation on the target instance. + * @param principal the principal being created/overwritten on the target Polaris instance + */ + void storePrincipal(PrincipalWithCredentials principal); + +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java index 0c6f4b10..398f7c3c 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -37,6 +38,53 @@ public AccessControlAwarePlanner(SynchronizationPlanner delegate) { super(delegate); } + @Override + public SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget) { + List skippedPrincipals = new ArrayList<>(); + List filteredPrincipalsSource = new ArrayList<>(); + List filteredPrincipalsTarget = new ArrayList<>(); + + for (Principal principal : principalsOnSource) { + if (principal.getProperties() != null + && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedPrincipals.add(principal); + continue; + } + + if (principal.getName().equals("root")) { + skippedPrincipals.add(principal); + continue; + } + + filteredPrincipalsSource.add(principal); + } + + for (Principal principal : principalsOnTarget) { + if (principal.getProperties() != null + && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedPrincipals.add(principal); + continue; + } + + if (principal.getName().equals("root")) { + skippedPrincipals.add(principal); + continue; + } + + filteredPrincipalsTarget.add(principal); + } + + SynchronizationPlan delegatedPlan = + delegate.planPrincipalSync(filteredPrincipalsSource, filteredPrincipalsTarget); + + for (Principal principal : skippedPrincipals) { + delegatedPlan.skipEntity(principal); + } + + return delegatedPlan; + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java index 67a38a21..6367f1b5 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -72,6 +73,20 @@ public class ModificationAwarePlanner implements SynchronizationPlanner { // GCP "storageConfigInfo.gcsServiceAccount"); + private static final String CLIENT_ID = "clientId"; + + private static final String CLIENT_SECRET = "clientSecret"; + + private static final List PRINCIPAL_KEYS_TO_IGNORE = List.of( + CREATE_TIMESTAMP, + LAST_UPDATE_TIMESTAMP, + ENTITY_VERSION, + + // client id and client secret will never be the same across the instances, ignore them + CLIENT_ID, + CLIENT_SECRET + ); + private final SynchronizationPlanner delegate; private final ObjectMapper objectMapper; @@ -142,6 +157,40 @@ private boolean areSame(Object o1, Object o2) { return areSame(o1, o2, DEFAULT_KEYS_TO_IGNORE); } + @Override + public SynchronizationPlan planPrincipalSync(List principalsOnSource, List principalsOnTarget) { + Map sourcePrincipalsByName = new HashMap<>(); + Map targetPrincipalsByName = new HashMap<>(); + + List notModifiedPrincipals = new ArrayList<>(); + + principalsOnSource.forEach(principal -> sourcePrincipalsByName.put(principal.getName(), principal)); + principalsOnTarget.forEach(principal -> targetPrincipalsByName.put(principal.getName(), principal)); + + for (Principal sourcePrincipal : principalsOnSource) { + if (targetPrincipalsByName.containsKey(sourcePrincipal.getName())) { + Principal targetPrincipal = targetPrincipalsByName.get(sourcePrincipal.getName()); + + if (areSame(sourcePrincipal, targetPrincipal, PRINCIPAL_KEYS_TO_IGNORE)) { + notModifiedPrincipals.add(targetPrincipal); + sourcePrincipalsByName.remove(sourcePrincipal.getName()); + targetPrincipalsByName.remove(targetPrincipal.getName()); + } + } + } + + SynchronizationPlan delegatedPlan = delegate.planPrincipalSync( + sourcePrincipalsByName.values().stream().toList(), + targetPrincipalsByName.values().stream().toList() + ); + + for (Principal principal : notModifiedPrincipals) { + delegatedPlan.skipEntityNotModified(principal); + } + + return delegatedPlan; + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java index 241febd0..290595e0 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java @@ -25,11 +25,18 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; public class NoOpSyncPlanner implements SynchronizationPlanner { + @Override + public SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget) { + return new SynchronizationPlan<>(); + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java index 622bfe25..be682d0b 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java @@ -26,6 +26,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -36,6 +37,34 @@ */ public class SourceParitySynchronizationPlanner implements SynchronizationPlanner { + @Override + public SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget) { + Set sourcePrincipalNames = principalsOnSource.stream().map(Principal::getName).collect(Collectors.toSet()); + Set targetPrincipalNames = principalsOnTarget.stream().map(Principal::getName).collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (Principal principal : principalsOnSource) { + if (targetPrincipalNames.contains(principal.getName())) { + // overwrite the entity on the target if it exists on the source + plan.overwriteEntity(principal); + } else { + // create the entity on the target if not exists + plan.createEntity(principal); + } + } + + for (Principal principal : principalsOnTarget) { + if (!sourcePrincipalNames.contains(principal.getName())) { + // remove the entity from the target if it doesn't exist on the source + plan.removeEntity(principal); + } + } + + return plan; + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java index 78edf9df..481d31ed 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -34,6 +35,9 @@ */ public interface SynchronizationPlanner { + SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget); + SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget); diff --git a/polaris-synchronizer/cli/build.gradle.kts b/polaris-synchronizer/cli/build.gradle.kts index e637d630..34cb4c63 100644 --- a/polaris-synchronizer/cli/build.gradle.kts +++ b/polaris-synchronizer/cli/build.gradle.kts @@ -63,6 +63,10 @@ tasks.named("build") { dependsOn("shadowJar") } +tasks.named("assemble") { + dependsOn("shadowJar") +} + // Optionally disable raw JAR tasks.named("jar") { enabled = false diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java new file mode 100644 index 00000000..c53a3d69 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java @@ -0,0 +1,91 @@ +package org.apache.polaris.tools.sync.polaris; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; +import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class CsvPrincipalCredentialsCaptor implements PrincipalCredentialCaptor, Closeable { + + private final static String PRINCIPAL_NAME_HEADER = "PrincipalName"; + + private final static String TARGET_CLIENT_ID_HEADER = "TargetClientId"; + + private final static String TARGET_CLIENT_SECRET_HEADER = "TargetClientSecret"; + + private final static String[] HEADERS = { + PRINCIPAL_NAME_HEADER, TARGET_CLIENT_ID_HEADER, TARGET_CLIENT_SECRET_HEADER }; + + private final File file; + + private final Map principalsByName; + + public CsvPrincipalCredentialsCaptor(final File file) throws IOException { + this.principalsByName = new HashMap<>(); + this.file = file; + + // We reload the file because we don't want to clear any existing data in case the same file + // is passed twice + if (this.file.exists()) { + CSVFormat readerCSVFormat = + CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get(); + + CSVParser parser = + CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat); + + for (CSVRecord record : parser.getRecords()) { + this.principalsByName.put( + record.get(PRINCIPAL_NAME_HEADER), + new PrincipalWithCredentials().credentials( + new PrincipalWithCredentialsCredentials() + .clientId(record.get(TARGET_CLIENT_ID_HEADER)) + .clientSecret(record.get(TARGET_CLIENT_SECRET_HEADER)))); + + } + + parser.close(); + } + } + + @Override + public void storePrincipal(PrincipalWithCredentials principal) { + this.principalsByName.put(principal.getPrincipal().getName(), principal); + } + + @Override + public void close() throws IOException { + BufferedWriter writer = Files.newBufferedWriter(file.toPath(), StandardCharsets.UTF_8); + + writer.write(""); // clear file + + CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS).get(); + + CSVPrinter csvPrinter = new CSVPrinter(writer, csvFormat); + + principalsByName.forEach((name, principal) -> { + try { + csvPrinter.printRecord(name, principal.getCredentials().getClientId(), principal.getCredentials().getClientSecret()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + csvPrinter.flush(); + csvPrinter.close(); + } + +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index eda18c58..7f416cc4 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.util.concurrent.Callable; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.access.NoOpPrincipalCredentialCaptor; +import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; import org.apache.polaris.tools.sync.polaris.catalog.ETagService; import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagService; import org.apache.polaris.tools.sync.polaris.options.SourceOmniPotentPrincipalOptions; @@ -76,6 +78,19 @@ public class SyncPolarisCommand implements Callable { description = "The file path of the file to retrieve and store table ETags from.") private String etagFilePath; + @CommandLine.Option( + names = {"--sync-principals"}, + description = "Enable principal synchronization. WARNING: Principal client-id and client-secret will be " + + "reset on the target Polaris instance." + ) + private boolean shouldSyncPrincipals; + + @CommandLine.Option( + names = {"--target-principal-credentials-file"}, + description = "The file path of the file to write credentials for principals created/overwritten on the " + + "target to.") + private String targetPrincipalCredentialsFilePath; + @Override public Integer call() throws Exception { SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner(); @@ -101,6 +116,26 @@ public Integer call() throws Exception { etagService = new NoOpETagService(); } + PrincipalCredentialCaptor principalCredentialCaptor; + + if (shouldSyncPrincipals) { + if (targetPrincipalCredentialsFilePath != null) { + consoleLog.warn("Principal creation will reset credentials on the target Polaris instance. " + + "Principal creation will produce a file {} with the collected Principal credentials on the target" + + " Polaris instance. Please ensure this file is stored securely as it contains sensitive credentials.", + targetPrincipalCredentialsFilePath); + + File targetCredentialFile = new File(targetPrincipalCredentialsFilePath); + principalCredentialCaptor = new CsvPrincipalCredentialsCaptor(targetCredentialFile); + } else { + consoleLog.error("Principal synchronization requires specifying the --target-principal-credentials-file option " + + "to store the reset Principal credentials on the target Polaris instance."); + return 1; + } + } else { + principalCredentialCaptor = new NoOpPrincipalCredentialCaptor(); + } + Runtime.getRuntime() .addShutdownHook( new Thread( @@ -112,6 +147,14 @@ public Integer call() throws Exception { throw new RuntimeException(e); } } + + if (principalCredentialCaptor instanceof Closeable closablePrincipalCredentialCaptor) { + try { + closablePrincipalCredentialCaptor.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } })); PolarisSynchronizer synchronizer = @@ -122,8 +165,12 @@ public Integer call() throws Exception { targetOmniPotentPrincipal, source, target, - etagService); + etagService, + principalCredentialCaptor); + if (shouldSyncPrincipals) { + synchronizer.syncPrincipals(); + } synchronizer.syncPrincipalRoles(); synchronizer.syncCatalogs(); From b24f648979bc097cae723abd95dd18983a35cf52 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Fri, 11 Apr 2025 16:28:43 -0700 Subject: [PATCH 03/18] Updated tests to accommodate principal creation --- .../planning/ModificationAwarePlanner.java | 7 ++- .../AccessControlAwarePlannerTest.java | 39 +++++++++++++++ .../polaris/ModificationAwarePlannerTest.java | 49 +++++++++++++++++++ ...ourceParitySynchronizationPlannerTest.java | 30 ++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java index 6367f1b5..01b9f687 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java @@ -82,9 +82,8 @@ public class ModificationAwarePlanner implements SynchronizationPlanner { LAST_UPDATE_TIMESTAMP, ENTITY_VERSION, - // client id and client secret will never be the same across the instances, ignore them - CLIENT_ID, - CLIENT_SECRET + // client id will never be the same across the instances, ignore it + CLIENT_ID ); private final SynchronizationPlanner delegate; @@ -172,7 +171,7 @@ public SynchronizationPlan planPrincipalSync(List principa Principal targetPrincipal = targetPrincipalsByName.get(sourcePrincipal.getName()); if (areSame(sourcePrincipal, targetPrincipal, PRINCIPAL_KEYS_TO_IGNORE)) { - notModifiedPrincipals.add(targetPrincipal); + notModifiedPrincipals.add(sourcePrincipal); sourcePrincipalsByName.remove(sourcePrincipal.getName()); targetPrincipalsByName.remove(targetPrincipal.getName()); } diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java index f40e6b1d..21542789 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants; import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; @@ -31,6 +32,44 @@ public class AccessControlAwarePlannerTest { + private static final Principal omnipotentPrincipalSource = + new Principal() + .name("omnipotent-principal-XXXXX") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + private static final Principal omnipotentPrincipalTarget = + new Principal() + .name("omnipotent-principal-YYYYY") + .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, ""); + + @Test + public void filtersOmnipotentPrincipal() { + SynchronizationPlanner accessControlAwarePlanner + = new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = accessControlAwarePlanner + .planPrincipalSync(List.of(omnipotentPrincipalSource), List.of(omnipotentPrincipalTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalTarget)); + } + + private static final Principal rootPrincipalSource = new Principal().name("root"); + + private static final Principal rootPrincipalTarget = new Principal().name("root"); + + @Test + public void filtersRootPrincipal() { + SynchronizationPlanner accessControlAwarePlanner + = new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = accessControlAwarePlanner + .planPrincipalSync(List.of(rootPrincipalSource), List.of(rootPrincipalTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(rootPrincipalSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(rootPrincipalTarget)); + } + private static final PrincipalRole omnipotentPrincipalRoleSource = new PrincipalRole() .name("omnipotent-principal-XXXXX") diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java index 5d875017..3e6b1072 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; @@ -39,6 +40,54 @@ public class ModificationAwarePlannerTest { + private static final Principal principal = new Principal().name("principal"); + + private static final Principal modifiedPrincipal = + new Principal().name("principal").putPropertiesItem("newproperty", "newvalue"); + + private static final Principal principalWithClientId = + new Principal() + .name("principal") + .clientId("clientId"); + + private static final Principal principalWithResetClientId = + new Principal() + .name("principal") + .clientId("clientIdNew"); + + @Test + public void testPrincipalNotModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planPrincipalSync(List.of(principal), List.of(principal)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(principal)); + } + + @Test + public void testPrincipalModified() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planPrincipalSync(List.of(principal), List.of(modifiedPrincipal)); + + Assertions.assertFalse(plan.entitiesNotModified().contains(principal)); + } + + @Test + public void testPrincipalNotModifiedWithResetClientId() { + SynchronizationPlanner modificationPlanner = + new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = + modificationPlanner.planPrincipalSync(List.of(principalWithClientId), List.of(principalWithResetClientId)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(principalWithClientId)); + } + private static final PrincipalRole principalRole = new PrincipalRole().name("principal-role"); private static final PrincipalRole modifiedPrincipalRole = diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java index c9a8480f..e20e9365 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -59,6 +60,35 @@ public void testCreatesNewCatalogOverwritesOldCatalogRemovesDroppedCatalog() { Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_3)); } + private static final Principal PRINCIPAL_1 = + new Principal().name("principal-1"); + + private static final Principal PRINCIPAL_2 = + new Principal().name("principal-2"); + + private static final Principal PRINCIPAL_3 = + new Principal().name("principal-3"); + + @Test + public void testCreatesNewPrincipalOverwritesOldPrincipalRemovesDroppedPrincipal() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planPrincipalSync(List.of(PRINCIPAL_1, PRINCIPAL_2), List.of(PRINCIPAL_2, PRINCIPAL_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_1)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_2)); + Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_3)); + } + private static final PrincipalRole PRINCIPAL_ROLE_1 = new PrincipalRole().name("principal-role-1"); From 2afaff4b3095ae591071f655aa5a37cd14241c22 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Sun, 13 Apr 2025 12:03:07 -0700 Subject: [PATCH 04/18] Added migration of principal roles to principals --- .../tools/sync/polaris/PolarisService.java | 6 +- .../sync/polaris/PolarisSynchronizer.java | 112 ++++++- .../polaris/access/AccessControlService.java | 2 +- .../access/NoOpPrincipalCredentialCaptor.java | 15 - .../access/PrincipalCredentialCaptor.java | 17 -- .../planning/AccessControlAwarePlanner.java | 62 +++- .../polaris/planning/DelegatedPlanner.java | 17 ++ .../planning/ModificationAwarePlanner.java | 244 +++++++++------- .../polaris/planning/NoOpSyncPlanner.java | 9 + .../SourceParitySynchronizationPlanner.java | 273 ++++++------------ .../planning/SynchronizationPlanner.java | 5 + .../AccessControlAwarePlannerTest.java | 8 +- .../CsvPrincipalCredentialsCaptor.java | 91 ------ .../sync/polaris/SyncPolarisCommand.java | 50 +--- 14 files changed, 446 insertions(+), 465 deletions(-) delete mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java delete mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java index 18ff23c3..03906bd4 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java @@ -93,12 +93,16 @@ public void removePrincipal(String principalName) { this.api.deletePrincipal(principalName); } - public void assignPrincipalRole(String principalName, String principalRoleName) { + public void assignPrincipalRoleToPrincipal(String principalName, String principalRoleName) { GrantPrincipalRoleRequest request = new GrantPrincipalRoleRequest().principalRole(new PrincipalRole().name(principalRoleName)); this.api.assignPrincipalRole(principalName, request); } + public void revokePrincipalRoleFromPrincipal(String principalName, String principalRoleName) { + this.api.revokePrincipalRole(principalName, principalRoleName); + } + public void createPrincipalRole(PrincipalRole principalRole, boolean overwrite) { if (overwrite) { removePrincipalRole(principalRole.getName()); diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index ca081ecb..56b3e447 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -34,7 +34,6 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.access.AccessControlService; -import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag; import org.apache.polaris.tools.sync.polaris.catalog.ETagService; import org.apache.polaris.tools.sync.polaris.catalog.NotModifiedException; @@ -70,8 +69,6 @@ public class PolarisSynchronizer { private final AccessControlService targetAccessControlService; - private final PrincipalCredentialCaptor principalCredentialCaptor; - private final ETagService etagService; public PolarisSynchronizer( @@ -81,8 +78,7 @@ public PolarisSynchronizer( PrincipalWithCredentials targetOmnipotentPrincipal, PolarisService source, PolarisService target, - ETagService etagService, - PrincipalCredentialCaptor principalCredentialCaptor) { + ETagService etagService) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.syncPlanner = synchronizationPlanner; @@ -100,7 +96,6 @@ public PolarisSynchronizer( targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal( targetOmnipotentPrincipal.getPrincipal().getName()); this.etagService = etagService; - this.principalCredentialCaptor = principalCredentialCaptor; } /** @@ -141,7 +136,7 @@ public void syncPrincipals() { syncPlanner.planPrincipalSync(principalsSource, principalsTarget); principalSyncPlan - .entitiesToSkip() + .entitiesToSkipAndSkipChildren() .forEach( principal -> clientLogger.info("Skipping principal {}.", principal.getName())); @@ -160,9 +155,13 @@ public void syncPrincipals() { for (Principal principal : principalSyncPlan.entitiesToCreate()) { try { PrincipalWithCredentials createdPrincipal = target.createPrincipal(principal, false /* overwrite */); - principalCredentialCaptor.storePrincipal(createdPrincipal); - clientLogger.info("Created principal {} on target. - {}/{}", - principal.getName(), ++syncsCompleted, totalSyncsToComplete); + clientLogger.info("Created principal {} on target. Target credentials: {}:{} - {}/{}", + principal.getName(), + createdPrincipal.getCredentials().getClientId(), + createdPrincipal.getCredentials().getClientSecret(), + ++syncsCompleted, + totalSyncsToComplete + ); } catch (Exception e) { clientLogger.error("Failed to create principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); @@ -172,9 +171,13 @@ public void syncPrincipals() { for (Principal principal : principalSyncPlan.entitiesToOverwrite()) { try { PrincipalWithCredentials overwrittenPrincipal = target.createPrincipal(principal, true /* overwrite */); - principalCredentialCaptor.storePrincipal(overwrittenPrincipal); - clientLogger.info("Overwrote principal {} on target. - {}/{}", - principal.getName(), ++syncsCompleted, totalSyncsToComplete); + clientLogger.info("Overwrote principal {} on target. Target credentials: {}:{} - {}/{}", + principal.getName(), + overwrittenPrincipal.getCredentials().getClientId(), + overwrittenPrincipal.getCredentials().getClientSecret(), + ++syncsCompleted, + totalSyncsToComplete + ); } catch (Exception e) { clientLogger.error("Failed to overwrite principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); @@ -191,6 +194,89 @@ public void syncPrincipals() { principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); } } + + for (Principal principal : principalSyncPlan.entitiesToSyncChildren()) { + syncAssignedPrincipalRolesForPrincipal(principal.getName()); + } + } + + public void syncAssignedPrincipalRolesForPrincipal(String principalName) { + List assignedPrincipalRolesSource; + + try { + assignedPrincipalRolesSource = source.listPrincipalRolesAssignedForPrincipal(principalName); + clientLogger.info("Listed {} assigned principal-roles for principal {} from source.", + assignedPrincipalRolesSource.size(), principalName); + } catch (Exception e) { + clientLogger.error("Failed to list assigned principal-roles for principal {} from source.", principalName, e); + return; + } + + List assignedPrincipalRolesTarget; + + try { + assignedPrincipalRolesTarget = target.listPrincipalRolesAssignedForPrincipal(principalName); + clientLogger.info("Listed {} assigned principal-roles for principal {} from target.", + assignedPrincipalRolesTarget.size(), principalName); + } catch (Exception e) { + clientLogger.error("Failed to list assigned principal-roles for principal {} from target.", principalName, e); + return; + } + + SynchronizationPlan assignedPrincipalRoleSyncPlan = + syncPlanner.planAssignPrincipalsToPrincipalRolesSync( + principalName, assignedPrincipalRolesSource, assignedPrincipalRolesTarget); + + assignedPrincipalRoleSyncPlan + .entitiesToSkip() + .forEach( + principalRole -> + clientLogger.info("Skipping assignment of principal-role {} to principal {}.", + principalName, principalRole.getName())); + + assignedPrincipalRoleSyncPlan + .entitiesNotModified() + .forEach( + principalRole -> + clientLogger.info( + "Principal {} is already assigned to principal-role {}, skipping.", + principalName, principalRole.getName())); + + int syncsCompleted = 0; + final int totalSyncsToComplete = totalSyncsToComplete(assignedPrincipalRoleSyncPlan); + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) { + try { + target.assignPrincipalRoleToPrincipal(principalName, principalRole.getName()); + clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } + } + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) { + try { + target.assignPrincipalRoleToPrincipal(principalName, principalRole.getName()); + clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } + } + + for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) { + try { + target.revokePrincipalRoleFromPrincipal(principalName, principalRole.getName()); + clientLogger.info("Revoked principal-role {} from principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error("Failed to revoke principal-role {} to principal {}. - {}/{}", + principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); + } + } } /** Sync principal roles from source to target. */ diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java index c0b4fe6e..28e83f31 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java @@ -149,7 +149,7 @@ public PrincipalRole createAndAssignPrincipalRole( } polaris.createPrincipalRole(omnipotentPrincipalRole, false); - polaris.assignPrincipalRole( + polaris.assignPrincipalRoleToPrincipal( omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName()); return omnipotentPrincipalRole; } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java deleted file mode 100644 index e589404f..00000000 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/NoOpPrincipalCredentialCaptor.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.polaris.tools.sync.polaris.access; - -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; - -/** - * No-op implementation that does nothing with the principal credentials. - */ -public class NoOpPrincipalCredentialCaptor implements PrincipalCredentialCaptor { - - @Override - public void storePrincipal(PrincipalWithCredentials principal) { - // do nothing - } - -} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java deleted file mode 100644 index 44b6741c..00000000 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/PrincipalCredentialCaptor.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.apache.polaris.tools.sync.polaris.access; - -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; - -/** - * Captures principal credentials at the time of a principal's - * creation/overwrite on the target Polaris instance. - */ -public interface PrincipalCredentialCaptor { - - /** - * Store the principal with associated credentials at the time of its creation on the target instance. - * @param principal the principal being created/overwritten on the target Polaris instance - */ - void storePrincipal(PrincipalWithCredentials principal); - -} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java index 398f7c3c..0e958b12 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java @@ -46,12 +46,14 @@ public SynchronizationPlan planPrincipalSync( List filteredPrincipalsTarget = new ArrayList<>(); for (Principal principal : principalsOnSource) { + // if the principal is the omnipotent principal do not sync it to target if (principal.getProperties() != null && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { skippedPrincipals.add(principal); continue; } + // do not modify root principal if (principal.getName().equals("root")) { skippedPrincipals.add(principal); continue; @@ -61,12 +63,15 @@ public SynchronizationPlan planPrincipalSync( } for (Principal principal : principalsOnTarget) { + // if the principal is the omnipotent principal, ensure it is not modified + // on the target if (principal.getProperties() != null && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { skippedPrincipals.add(principal); continue; } + // do not modify root principal if (principal.getName().equals("root")) { skippedPrincipals.add(principal); continue; @@ -79,7 +84,62 @@ public SynchronizationPlan planPrincipalSync( delegate.planPrincipalSync(filteredPrincipalsSource, filteredPrincipalsTarget); for (Principal principal : skippedPrincipals) { - delegatedPlan.skipEntity(principal); + delegatedPlan.skipEntityAndSkipChildren(principal); + } + + return delegatedPlan; + } + + @Override + public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget + ) { + List skippedRoles = new ArrayList<>(); + List filteredRolesSource = new ArrayList<>(); + List filteredRolesTarget = new ArrayList<>(); + + for (PrincipalRole role : assignedPrincipalRolesOnSource) { + // filter out assignment to omnipotent principal role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out assignment to service admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesSource.add(role); + } + + for (PrincipalRole role : assignedPrincipalRolesOnTarget) { + // filer out assignment to omnipotent principal role + if (role.getProperties() != null + && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) { + skippedRoles.add(role); + continue; + } + + // filter out assignment to service admin + if (role.getName().equals("service_admin")) { + skippedRoles.add(role); + continue; + } + + filteredRolesTarget.add(role); + } + + SynchronizationPlan delegatedPlan = + this.delegate.planAssignPrincipalsToPrincipalRolesSync( + principalName, filteredRolesSource, filteredRolesTarget); + + for (PrincipalRole role : skippedRoles) { + delegatedPlan.skipEntity(role); } return delegatedPlan; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java index e48b5323..2db30159 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -40,6 +41,22 @@ public DelegatedPlanner(SynchronizationPlanner delegate) { this.delegate = delegate; } + @Override + public SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget) { + return delegate.planPrincipalSync(principalsOnSource, principalsOnTarget); + } + + @Override + public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget + ) { + return delegate.planAssignPrincipalsToPrincipalRolesSync( + principalName, assignedPrincipalRolesOnSource, assignedPrincipalRolesOnTarget); + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java index 01b9f687..ae2bf4a8 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java @@ -21,12 +21,17 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; + import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.admin.model.Catalog; @@ -35,6 +40,7 @@ import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.eclipse.collections.impl.block.procedure.CollectIfProcedure; /** Planner that checks for modifications and plans to skip entities that have not been modified. */ public class ModificationAwarePlanner implements SynchronizationPlanner { @@ -156,34 +162,79 @@ private boolean areSame(Object o1, Object o2) { return areSame(o1, o2, DEFAULT_KEYS_TO_IGNORE); } - @Override - public SynchronizationPlan planPrincipalSync(List principalsOnSource, List principalsOnTarget) { - Map sourcePrincipalsByName = new HashMap<>(); - Map targetPrincipalsByName = new HashMap<>(); - - List notModifiedPrincipals = new ArrayList<>(); - - principalsOnSource.forEach(principal -> sourcePrincipalsByName.put(principal.getName(), principal)); - principalsOnTarget.forEach(principal -> targetPrincipalsByName.put(principal.getName(), principal)); - - for (Principal sourcePrincipal : principalsOnSource) { - if (targetPrincipalsByName.containsKey(sourcePrincipal.getName())) { - Principal targetPrincipal = targetPrincipalsByName.get(sourcePrincipal.getName()); + /** + * Container to represent the result of filtering out entities that are on both the source and the target + * and have not been modified. + * @param filteredEntitiesSource entities on the source that are do not exist on the + * target or are different from the target for the same identifier + * @param filteredEntitiesTarget entities on the target that do not exist on the source or are different from the + * source for the same identifier + * @param notModifiedEntities entities from the source that have not been modified since migration to the target + * @param the entity type + */ + private record FilteredNotModifiedEntityResult( + List filteredEntitiesSource, List filteredEntitiesTarget, List notModifiedEntities) {} - if (areSame(sourcePrincipal, targetPrincipal, PRINCIPAL_KEYS_TO_IGNORE)) { - notModifiedPrincipals.add(sourcePrincipal); - sourcePrincipalsByName.remove(sourcePrincipal.getName()); - targetPrincipalsByName.remove(targetPrincipal.getName()); + /** + * Filter out entities that are the same across the source and target Polaris instance. + * @param entitiesOnSource the entities from the source + * @param entitiesOnTarget the entities from the target + * @param entityIdentifierSupplier supplies an identifier to identify the same entity on the source and target + * @param entitiesAreSame compares entity with same identifier on source and target, returns true if they are the same, + * false otherwise + * @return the entities on the source and target with unmodified entities filtered out + * @param the entity type + */ + private FilteredNotModifiedEntityResult filterOutEntitiesNotModified( + Collection entitiesOnSource, + Collection entitiesOnTarget, + Function entityIdentifierSupplier, + BiFunction entitiesAreSame + ) { + Map sourceEntitiesById = new HashMap<>(); + Map targetEntitiesById = new HashMap<>(); + + List notModifiedEntities = new ArrayList<>(); + + entitiesOnSource.forEach(entity -> sourceEntitiesById.put(entityIdentifierSupplier.apply(entity), entity)); + entitiesOnTarget.forEach(entity -> targetEntitiesById.put(entityIdentifierSupplier.apply(entity), entity)); + + for (T sourceEntity : entitiesOnSource) { + Object sourceEntityId = entityIdentifierSupplier.apply(sourceEntity); + if (targetEntitiesById.containsKey(sourceEntityId)) { + T targetEntity = targetEntitiesById.get(sourceEntityId); + Object targetEntityId = entityIdentifierSupplier.apply(targetEntity); + + if (entitiesAreSame.apply(sourceEntity, targetEntity)) { + notModifiedEntities.add(sourceEntity); + sourceEntitiesById.remove(sourceEntityId); + targetEntitiesById.remove(targetEntityId); } } } + return new FilteredNotModifiedEntityResult<>( + sourceEntitiesById.values().stream().toList(), + targetEntitiesById.values().stream().toList(), + notModifiedEntities + ); + } + + @Override + public SynchronizationPlan planPrincipalSync(List principalsOnSource, List principalsOnTarget) { + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + principalsOnSource, + principalsOnTarget, + Principal::getName, + (p1, p2) -> areSame(p1, p2, PRINCIPAL_KEYS_TO_IGNORE) + ); + SynchronizationPlan delegatedPlan = delegate.planPrincipalSync( - sourcePrincipalsByName.values().stream().toList(), - targetPrincipalsByName.values().stream().toList() + result.filteredEntitiesSource(), + result.filteredEntitiesTarget() ); - for (Principal principal : notModifiedPrincipals) { + for (Principal principal : result.notModifiedEntities()) { delegatedPlan.skipEntityNotModified(principal); } @@ -191,34 +242,47 @@ public SynchronizationPlan planPrincipalSync(List principa } @Override - public SynchronizationPlan planPrincipalRoleSync( - List principalRolesOnSource, List principalRolesOnTarget) { - Map sourceRolesByName = new HashMap<>(); - Map targetRolesByName = new HashMap<>(); + public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget + ) { + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget, + PrincipalRole::getName, + this::areSame + ); - List notModifiedPrincipalRoles = new ArrayList<>(); + SynchronizationPlan delegatedPlan = + delegate.planAssignPrincipalsToPrincipalRolesSync( + principalName, + result.filteredEntitiesSource(), + result.filteredEntitiesTarget()); - principalRolesOnSource.forEach(role -> sourceRolesByName.put(role.getName(), role)); - principalRolesOnTarget.forEach(role -> targetRolesByName.put(role.getName(), role)); + for (PrincipalRole principalRole : result.notModifiedEntities()) { + delegatedPlan.skipEntityNotModified(principalRole); + } - for (PrincipalRole sourceRole : principalRolesOnSource) { - if (targetRolesByName.containsKey(sourceRole.getName())) { - PrincipalRole targetRole = targetRolesByName.get(sourceRole.getName()); + return delegatedPlan; + } - if (areSame(sourceRole, targetRole)) { - targetRolesByName.remove(targetRole.getName()); - sourceRolesByName.remove(sourceRole.getName()); - notModifiedPrincipalRoles.add(sourceRole); - } - } - } + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + principalRolesOnSource, + principalRolesOnTarget, + PrincipalRole::getName, + this::areSame + ); SynchronizationPlan delegatedPlan = delegate.planPrincipalRoleSync( - sourceRolesByName.values().stream().toList(), - targetRolesByName.values().stream().toList()); + result.filteredEntitiesSource(), + result.filteredEntitiesTarget()); - for (PrincipalRole principalRole : notModifiedPrincipalRoles) { + for (PrincipalRole principalRole : result.notModifiedEntities()) { delegatedPlan.skipEntityNotModified(principalRole); } @@ -240,32 +304,19 @@ private boolean areSame(Catalog source, Catalog target) { @Override public SynchronizationPlan planCatalogSync( List catalogsOnSource, List catalogsOnTarget) { - Map sourceCatalogsByName = new HashMap<>(); - Map targetCatalogsByName = new HashMap<>(); - - List notModifiedCatalogs = new ArrayList<>(); - - catalogsOnSource.forEach(catalog -> sourceCatalogsByName.put(catalog.getName(), catalog)); - catalogsOnTarget.forEach(catalog -> targetCatalogsByName.put(catalog.getName(), catalog)); - - for (Catalog sourceCatalog : catalogsOnSource) { - if (targetCatalogsByName.containsKey(sourceCatalog.getName())) { - Catalog targetCatalog = targetCatalogsByName.get(sourceCatalog.getName()); - - if (areSame(sourceCatalog, targetCatalog)) { - targetCatalogsByName.remove(targetCatalog.getName()); - sourceCatalogsByName.remove(sourceCatalog.getName()); - notModifiedCatalogs.add(sourceCatalog); - } - } - } + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + catalogsOnSource, + catalogsOnTarget, + Catalog::getName, + this::areSame + ); SynchronizationPlan delegatedPlan = delegate.planCatalogSync( - sourceCatalogsByName.values().stream().toList(), - targetCatalogsByName.values().stream().toList()); + result.filteredEntitiesSource(), + result.filteredEntitiesTarget()); - for (Catalog catalog : notModifiedCatalogs) { + for (Catalog catalog : result.notModifiedEntities()) { delegatedPlan.skipEntityNotModified(catalog); } @@ -277,33 +328,20 @@ public SynchronizationPlan planCatalogRoleSync( String catalogName, List catalogRolesOnSource, List catalogRolesOnTarget) { - Map sourceCatalogRolesByName = new HashMap<>(); - Map targetCatalogRolesByName = new HashMap<>(); - - List notModifiedCatalogRoles = new ArrayList<>(); - - catalogRolesOnSource.forEach(role -> sourceCatalogRolesByName.put(role.getName(), role)); - catalogRolesOnTarget.forEach(role -> targetCatalogRolesByName.put(role.getName(), role)); - - for (CatalogRole sourceCatalogRole : catalogRolesOnSource) { - if (targetCatalogRolesByName.containsKey(sourceCatalogRole.getName())) { - CatalogRole targetCatalogRole = targetCatalogRolesByName.get(sourceCatalogRole.getName()); - - if (areSame(sourceCatalogRole, targetCatalogRole)) { - targetCatalogRolesByName.remove(targetCatalogRole.getName()); - sourceCatalogRolesByName.remove(sourceCatalogRole.getName()); - notModifiedCatalogRoles.add(sourceCatalogRole); - } - } - } + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + catalogRolesOnSource, + catalogRolesOnTarget, + CatalogRole::getName, + this::areSame + ); SynchronizationPlan delegatedPlan = delegate.planCatalogRoleSync( catalogName, - sourceCatalogRolesByName.values().stream().toList(), - targetCatalogRolesByName.values().stream().toList()); + result.filteredEntitiesSource(), + result.filteredEntitiesTarget()); - for (CatalogRole catalogRole : notModifiedCatalogRoles) { + for (CatalogRole catalogRole : result.notModifiedEntities()) { delegatedPlan.skipEntityNotModified(catalogRole); } @@ -316,27 +354,21 @@ public SynchronizationPlan planGrantSync( String catalogRoleName, List grantsOnSource, List grantsOnTarget) { - Set sourceGrants = new HashSet<>(grantsOnSource); - Set targetGrants = new HashSet<>(grantsOnTarget); - - List notModifiedGrants = new ArrayList<>(); - - for (GrantResource grantResource : grantsOnSource) { - if (targetGrants.contains(grantResource)) { - sourceGrants.remove(grantResource); - targetGrants.remove(grantResource); - notModifiedGrants.add(grantResource); - } - } + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + grantsOnSource, + grantsOnTarget, + grant -> grant, + GrantResource::equals + ); SynchronizationPlan delegatedPlan = delegate.planGrantSync( catalogName, catalogRoleName, - sourceGrants.stream().toList(), - targetGrants.stream().toList()); + result.filteredEntitiesSource(), + result.filteredEntitiesTarget()); - for (GrantResource grant : notModifiedGrants) { + for (GrantResource grant : result.notModifiedEntities()) { delegatedPlan.skipEntityNotModified(grant); } @@ -349,11 +381,23 @@ public SynchronizationPlan planAssignPrincipalRolesToCatalogRoles String catalogRoleName, List assignedPrincipalRolesOnSource, List assignedPrincipalRolesOnTarget) { - return delegate.planAssignPrincipalRolesToCatalogRolesSync( + FilteredNotModifiedEntityResult result = filterOutEntitiesNotModified( + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget, + PrincipalRole::getName, + this::areSame + ); + + SynchronizationPlan delegatedPlan = delegate.planAssignPrincipalRolesToCatalogRolesSync( catalogName, catalogRoleName, assignedPrincipalRolesOnSource, assignedPrincipalRolesOnTarget); + + for (PrincipalRole principalRole : result.notModifiedEntities()) { + delegatedPlan.skipEntityNotModified(principalRole); + } + return delegatedPlan; } @Override diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java index 290595e0..82da075f 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java @@ -37,6 +37,15 @@ public SynchronizationPlan planPrincipalSync( return new SynchronizationPlan<>(); } + @Override + public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget + ) { + return new SynchronizationPlan<>(); + } + @Override public SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java index be682d0b..a60fea7d 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java @@ -18,8 +18,10 @@ */ package org.apache.polaris.tools.sync.polaris.planning; +import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -37,28 +39,57 @@ */ public class SourceParitySynchronizationPlanner implements SynchronizationPlanner { - @Override - public SynchronizationPlan planPrincipalSync( - List principalsOnSource, List principalsOnTarget) { - Set sourcePrincipalNames = principalsOnSource.stream().map(Principal::getName).collect(Collectors.toSet()); - Set targetPrincipalNames = principalsOnTarget.stream().map(Principal::getName).collect(Collectors.toSet()); - - SynchronizationPlan plan = new SynchronizationPlan<>(); - - for (Principal principal : principalsOnSource) { - if (targetPrincipalNames.contains(principal.getName())) { - // overwrite the entity on the target if it exists on the source - plan.overwriteEntity(principal); + /** + * Sort entities from the source into create, overwrite, and remove categories + * on the basis of which identifiers exist on the source and target Polaris. + * Identifiers that are both on the source and target instance will be marked + * for overwrite. Entities that are only on the source instance will be marked for + * creation. Entities that are only on the target instance will be marked for deletion. + * @param entitiesOnSource the entities from the source + * @param entitiesOnTarget the entities from the target + * @param supportOverwrites true if "overwriting" the entity is necessary. Most grant record entities do not need overwriting. + * @param entityIdentifierSupplier consumes an entity and returns an identifying representation of that entity + * @return a {@link SynchronizationPlan} with the entities sorted based on the souce parity strategy + * @param the type of the entity + */ + private SynchronizationPlan sortOnIdentifier( + Collection entitiesOnSource, + Collection entitiesOnTarget, + boolean supportOverwrites, + Function entityIdentifierSupplier + ) { + Set sourceEntityIdentifiers = entitiesOnSource.stream().map(entityIdentifierSupplier).collect(Collectors.toSet()); + Set targetEntityIdentifiers = entitiesOnTarget.stream().map(entityIdentifierSupplier).collect(Collectors.toSet()); + + SynchronizationPlan plan = new SynchronizationPlan<>(); + + for (T entityOnSource : entitiesOnSource) { + Object sourceEntityId = entityIdentifierSupplier.apply(entityOnSource); + if (targetEntityIdentifiers.contains(sourceEntityId)) { + if (supportOverwrites) { + // if the same entity identifier is on the source and the target, + // overwrite the one on the target with the one on the source + plan.overwriteEntity(entityOnSource); + } } else { - // create the entity on the target if not exists - plan.createEntity(principal); + // if the entity identifier only exists on the source, that means + // we need to create it for the first time on the target + plan.createEntity(entityOnSource); } } - for (Principal principal : principalsOnTarget) { - if (!sourcePrincipalNames.contains(principal.getName())) { - // remove the entity from the target if it doesn't exist on the source - plan.removeEntity(principal); + for (T entityOnTarget : entitiesOnTarget) { + Object targetEntityId = entityIdentifierSupplier.apply(entityOnTarget); + if (!sourceEntityIdentifiers.contains(targetEntityId)) { + // if the entity exists on the target but doesn't exist on the source, + // clean it up from the target + + // this is especially important for access control entities, as, for example, + // we could have a scenario where a grant was revoked from a catalog role, + // or a catalog role was revoked from a principal role, in which case the target + // should reflect this change when the tool is run multiple times, because we don't + // want to take chances with over-extending privileges + plan.removeEntity(entityOnTarget); } } @@ -66,63 +97,41 @@ public SynchronizationPlan planPrincipalSync( } @Override - public SynchronizationPlan planPrincipalRoleSync( - List principalRolesOnSource, List principalRolesOnTarget) { - Set sourcePrincipalRoleNames = - principalRolesOnSource.stream().map(PrincipalRole::getName).collect(Collectors.toSet()); - Set targetPrincipalRoleNames = - principalRolesOnTarget.stream().map(PrincipalRole::getName).collect(Collectors.toSet()); - - SynchronizationPlan plan = new SynchronizationPlan<>(); + public SynchronizationPlan planPrincipalSync( + List principalsOnSource, List principalsOnTarget) { + return sortOnIdentifier(principalsOnSource, principalsOnTarget, /* supportsOverwrites */ true, Principal::getName); + } - for (PrincipalRole principalRole : principalRolesOnSource) { - if (targetPrincipalRoleNames.contains(principalRole.getName())) { - // overwrite roles that exist on both - plan.overwriteEntity(principalRole); - } else { - // create roles on target that only exist on source - plan.createEntity(principalRole); - } - } + @Override + public SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget + ) { + return sortOnIdentifier( + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget, + /* supportsOverwrites */ false, // do not need to overwrite an assignment of a principal role to a principal + PrincipalRole::getName + ); + } - // remove roles that aren't on source - for (PrincipalRole principalRole : principalRolesOnTarget) { - if (!sourcePrincipalRoleNames.contains(principalRole.getName())) { - plan.removeEntity(principalRole); - } - } + @Override + public SynchronizationPlan planPrincipalRoleSync( + List principalRolesOnSource, List principalRolesOnTarget) { - return plan; + return sortOnIdentifier( + principalRolesOnSource, + principalRolesOnTarget, + /* supportsOverwrites */ true, + PrincipalRole::getName + ); } @Override public SynchronizationPlan planCatalogSync( List catalogsOnSource, List catalogsOnTarget) { - Set sourceCatalogNames = - catalogsOnSource.stream().map(Catalog::getName).collect(Collectors.toSet()); - Set targetCatalogNames = - catalogsOnTarget.stream().map(Catalog::getName).collect(Collectors.toSet()); - - SynchronizationPlan plan = new SynchronizationPlan<>(); - - for (Catalog catalog : catalogsOnSource) { - if (targetCatalogNames.contains(catalog.getName())) { - // overwrite catalogs on target that exist on both - plan.overwriteEntity(catalog); - } else { - // create catalogs on target that exist only on source - plan.createEntity(catalog); - } - } - - // remove catalogs that are only on target - for (Catalog catalog : catalogsOnTarget) { - if (!sourceCatalogNames.contains(catalog.getName())) { - plan.removeEntity(catalog); - } - } - - return plan; + return sortOnIdentifier(catalogsOnSource, catalogsOnTarget, /* supportsOverwrites */ true, Catalog::getName); } @Override @@ -130,31 +139,8 @@ public SynchronizationPlan planCatalogRoleSync( String catalogName, List catalogRolesOnSource, List catalogRolesOnTarget) { - Set sourceCatalogRoleNames = - catalogRolesOnSource.stream().map(CatalogRole::getName).collect(Collectors.toSet()); - Set targetCatalogRoleNames = - catalogRolesOnTarget.stream().map(CatalogRole::getName).collect(Collectors.toSet()); - - SynchronizationPlan plan = new SynchronizationPlan<>(); - - for (CatalogRole catalogRole : catalogRolesOnSource) { - if (targetCatalogRoleNames.contains(catalogRole.getName())) { - plan.overwriteEntity(catalogRole); - // overwrite catalog roles on both - } else { - // create catalog roles on target that are only on source - plan.createEntity(catalogRole); - } - } - - // remove catalog roles on both the source and target - for (CatalogRole catalogRole : catalogRolesOnTarget) { - if (!sourceCatalogRoleNames.contains(catalogRole.getName())) { - plan.removeEntity(catalogRole); - } - } - - return plan; + return sortOnIdentifier( + catalogRolesOnSource, catalogRolesOnTarget, /* supportsOverwrites */ true, CatalogRole::getName); } @Override @@ -163,27 +149,12 @@ public SynchronizationPlan planGrantSync( String catalogRoleName, List grantsOnSource, List grantsOnTarget) { - Set grantsSourceSet = Set.copyOf(grantsOnSource); - Set grantsTargetSet = Set.copyOf(grantsOnTarget); - - SynchronizationPlan plan = new SynchronizationPlan<>(); - - // special case: no concept of overwriting a grant - // it exists and cannot change, so just create new ones - for (GrantResource grant : grantsOnSource) { - if (!grantsTargetSet.contains(grant)) { - plan.createEntity(grant); - } - } - - // remove grants that are not on the source - for (GrantResource grant : grantsOnTarget) { - if (!grantsSourceSet.contains(grant)) { - plan.removeEntity(grant); - } - } - - return plan; + return sortOnIdentifier( + grantsOnSource, + grantsOnTarget, + /* supportsOverwrites */ false, + grant -> grant // grants can just be compared by the entire generated object + ); } @Override @@ -192,33 +163,12 @@ public SynchronizationPlan planAssignPrincipalRolesToCatalogRoles String catalogRoleName, List assignedPrincipalRolesOnSource, List assignedPrincipalRolesOnTarget) { - Set sourcePrincipalRoleNames = - assignedPrincipalRolesOnSource.stream() - .map(PrincipalRole::getName) - .collect(Collectors.toSet()); - Set targetPrincipalRoleNames = - assignedPrincipalRolesOnTarget.stream() - .map(PrincipalRole::getName) - .collect(Collectors.toSet()); - - SynchronizationPlan plan = new SynchronizationPlan<>(); - - // special case: no concept of overwriting an assignment of principal role to catalog role - // it either exists or it doesn't, it cannot change - for (PrincipalRole principalRole : assignedPrincipalRolesOnSource) { - if (!targetPrincipalRoleNames.contains(principalRole.getName())) { - plan.createEntity(principalRole); - } - } - - // revoke principal roles that do not exist on the source - for (PrincipalRole principalRole : assignedPrincipalRolesOnTarget) { - if (!sourcePrincipalRoleNames.contains(principalRole.getName())) { - plan.removeEntity(principalRole); - } - } - - return plan; + return sortOnIdentifier( + assignedPrincipalRolesOnSource, + assignedPrincipalRolesOnTarget, + /* supportsOverwrites */ false, + PrincipalRole::getName + ); } @Override @@ -227,26 +177,7 @@ public SynchronizationPlan planNamespaceSync( Namespace namespace, List namespacesOnSource, List namespacesOnTarget) { - SynchronizationPlan plan = new SynchronizationPlan<>(); - - for (Namespace ns : namespacesOnSource) { - if (namespacesOnTarget.contains(ns)) { - // overwrite the entity on the target with the entity on the source - plan.overwriteEntity(ns); - } else { - // if the namespace is not on the target, plan to create it - plan.createEntity(ns); - } - } - - for (Namespace ns : namespacesOnTarget) { - if (!namespacesOnSource.contains(ns)) { - // remove namespaces that do not exist on the source but do exist on the target - plan.removeEntity(ns); - } - } - - return plan; + return sortOnIdentifier(namespacesOnSource, namespacesOnTarget, /* supportsOverwrites */ true, ns -> ns); } @Override @@ -255,25 +186,7 @@ public SynchronizationPlan planTableSync( Namespace namespace, Set tablesOnSource, Set tablesOnTarget) { - SynchronizationPlan plan = new SynchronizationPlan<>(); - - for (TableIdentifier tableIdentifier : tablesOnSource) { - if (tablesOnTarget.contains(tableIdentifier)) { - // overwrite tables on target and source - plan.overwriteEntity(tableIdentifier); - } else { - // create tables on source but not target - plan.createEntity(tableIdentifier); - } - } - - // remove tables only on target - for (TableIdentifier tableIdentifier : tablesOnTarget) { - if (!tablesOnSource.contains(tableIdentifier)) { - plan.removeEntity(tableIdentifier); - } - } - - return plan; + return sortOnIdentifier( + tablesOnSource, tablesOnTarget, /* supportsOverwrites */ true, tableId -> tableId); } } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java index 481d31ed..93cddab7 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java @@ -38,6 +38,11 @@ public interface SynchronizationPlanner { SynchronizationPlan planPrincipalSync( List principalsOnSource, List principalsOnTarget); + SynchronizationPlan planAssignPrincipalsToPrincipalRolesSync( + String principalName, + List assignedPrincipalRolesOnSource, + List assignedPrincipalRolesOnTarget); + SynchronizationPlan planPrincipalRoleSync( List principalRolesOnSource, List principalRolesOnTarget); diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java index 21542789..51d459ad 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java @@ -50,8 +50,8 @@ public void filtersOmnipotentPrincipal() { SynchronizationPlan plan = accessControlAwarePlanner .planPrincipalSync(List.of(omnipotentPrincipalSource), List.of(omnipotentPrincipalTarget)); - Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalSource)); - Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalTarget)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(omnipotentPrincipalSource)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(omnipotentPrincipalTarget)); } private static final Principal rootPrincipalSource = new Principal().name("root"); @@ -66,8 +66,8 @@ public void filtersRootPrincipal() { SynchronizationPlan plan = accessControlAwarePlanner .planPrincipalSync(List.of(rootPrincipalSource), List.of(rootPrincipalTarget)); - Assertions.assertTrue(plan.entitiesToSkip().contains(rootPrincipalSource)); - Assertions.assertTrue(plan.entitiesToSkip().contains(rootPrincipalTarget)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(rootPrincipalSource)); + Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(rootPrincipalTarget)); } private static final PrincipalRole omnipotentPrincipalRoleSource = diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java deleted file mode 100644 index c53a3d69..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvPrincipalCredentialsCaptor.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.apache.polaris.tools.sync.polaris; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; -import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; - -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.Map; - -import static java.nio.charset.StandardCharsets.UTF_8; - -public class CsvPrincipalCredentialsCaptor implements PrincipalCredentialCaptor, Closeable { - - private final static String PRINCIPAL_NAME_HEADER = "PrincipalName"; - - private final static String TARGET_CLIENT_ID_HEADER = "TargetClientId"; - - private final static String TARGET_CLIENT_SECRET_HEADER = "TargetClientSecret"; - - private final static String[] HEADERS = { - PRINCIPAL_NAME_HEADER, TARGET_CLIENT_ID_HEADER, TARGET_CLIENT_SECRET_HEADER }; - - private final File file; - - private final Map principalsByName; - - public CsvPrincipalCredentialsCaptor(final File file) throws IOException { - this.principalsByName = new HashMap<>(); - this.file = file; - - // We reload the file because we don't want to clear any existing data in case the same file - // is passed twice - if (this.file.exists()) { - CSVFormat readerCSVFormat = - CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get(); - - CSVParser parser = - CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat); - - for (CSVRecord record : parser.getRecords()) { - this.principalsByName.put( - record.get(PRINCIPAL_NAME_HEADER), - new PrincipalWithCredentials().credentials( - new PrincipalWithCredentialsCredentials() - .clientId(record.get(TARGET_CLIENT_ID_HEADER)) - .clientSecret(record.get(TARGET_CLIENT_SECRET_HEADER)))); - - } - - parser.close(); - } - } - - @Override - public void storePrincipal(PrincipalWithCredentials principal) { - this.principalsByName.put(principal.getPrincipal().getName(), principal); - } - - @Override - public void close() throws IOException { - BufferedWriter writer = Files.newBufferedWriter(file.toPath(), StandardCharsets.UTF_8); - - writer.write(""); // clear file - - CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS).get(); - - CSVPrinter csvPrinter = new CSVPrinter(writer, csvFormat); - - principalsByName.forEach((name, principal) -> { - try { - csvPrinter.printRecord(name, principal.getCredentials().getClientId(), principal.getCredentials().getClientSecret()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - csvPrinter.flush(); - csvPrinter.close(); - } - -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 7f416cc4..c4f0e043 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.util.concurrent.Callable; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.tools.sync.polaris.access.NoOpPrincipalCredentialCaptor; -import org.apache.polaris.tools.sync.polaris.access.PrincipalCredentialCaptor; import org.apache.polaris.tools.sync.polaris.catalog.ETagService; import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagService; import org.apache.polaris.tools.sync.polaris.options.SourceOmniPotentPrincipalOptions; @@ -80,17 +78,13 @@ public class SyncPolarisCommand implements Callable { @CommandLine.Option( names = {"--sync-principals"}, - description = "Enable principal synchronization. WARNING: Principal client-id and client-secret will be " + - "reset on the target Polaris instance." + description = "Enable synchronization of principals across the source and target, and assign them to " + + "the appropriate principal roles. WARNING: Principal client-id and client-secret will be reset on " + + "the target Polaris instance, and the new credentials for the principals created on the target will " + + "be logged to stdout." ) private boolean shouldSyncPrincipals; - @CommandLine.Option( - names = {"--target-principal-credentials-file"}, - description = "The file path of the file to write credentials for principals created/overwritten on the " + - "target to.") - private String targetPrincipalCredentialsFilePath; - @Override public Integer call() throws Exception { SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner(); @@ -116,26 +110,6 @@ public Integer call() throws Exception { etagService = new NoOpETagService(); } - PrincipalCredentialCaptor principalCredentialCaptor; - - if (shouldSyncPrincipals) { - if (targetPrincipalCredentialsFilePath != null) { - consoleLog.warn("Principal creation will reset credentials on the target Polaris instance. " + - "Principal creation will produce a file {} with the collected Principal credentials on the target" + - " Polaris instance. Please ensure this file is stored securely as it contains sensitive credentials.", - targetPrincipalCredentialsFilePath); - - File targetCredentialFile = new File(targetPrincipalCredentialsFilePath); - principalCredentialCaptor = new CsvPrincipalCredentialsCaptor(targetCredentialFile); - } else { - consoleLog.error("Principal synchronization requires specifying the --target-principal-credentials-file option " + - "to store the reset Principal credentials on the target Polaris instance."); - return 1; - } - } else { - principalCredentialCaptor = new NoOpPrincipalCredentialCaptor(); - } - Runtime.getRuntime() .addShutdownHook( new Thread( @@ -147,14 +121,6 @@ public Integer call() throws Exception { throw new RuntimeException(e); } } - - if (principalCredentialCaptor instanceof Closeable closablePrincipalCredentialCaptor) { - try { - closablePrincipalCredentialCaptor.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } })); PolarisSynchronizer synchronizer = @@ -165,13 +131,13 @@ public Integer call() throws Exception { targetOmniPotentPrincipal, source, target, - etagService, - principalCredentialCaptor); - + etagService); + synchronizer.syncPrincipalRoles(); if (shouldSyncPrincipals) { + consoleLog.warn("Principal migration will reset credentials on the target Polaris instance. " + + "Principal migration will log the new target Principal credentials to stdout."); synchronizer.syncPrincipals(); } - synchronizer.syncPrincipalRoles(); synchronizer.syncCatalogs(); return 0; From 05f2988edb59fe44740c0e6a6a038129689cde36 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Sun, 13 Apr 2025 15:17:13 -0700 Subject: [PATCH 05/18] Addressed comments --- polaris-synchronizer/README.md | 25 +++++++++++-------- .../sync/polaris/PolarisSynchronizer.java | 22 +++++++++------- .../access/AccessControlConstants.java | 10 ++++++++ .../polaris/catalog/BaseTableWithETag.java | 7 +++++- .../{ETagService.java => ETagManager.java} | 2 +- ...java => MetadataNotModifiedException.java} | 14 ++++++++--- .../MetadataWrapperTableOperations.java | 3 +++ ...pETagService.java => NoOpETagManager.java} | 2 +- .../sync/polaris/catalog/PolarisCatalog.java | 7 ++++-- .../polaris/planning/NoOpSyncPlanner.java | 3 +++ .../CreateOmnipotentPrincipalCommand.java | 4 +++ ...svETagService.java => CsvETagManager.java} | 6 ++--- .../sync/polaris/PolarisSynchronizerCLI.java | 5 +++- .../sync/polaris/SyncPolarisCommand.java | 13 ++++++---- .../BaseOmnipotentPrincipalOptions.java | 5 ++++ .../polaris/options/BasePolarisOptions.java | 5 ++++ .../sync/polaris/options/PolarisOptions.java | 4 +++ .../SourceOmniPotentPrincipalOptions.java | 4 +++ .../polaris/options/SourcePolarisOptions.java | 4 +++ .../options/TargetOmnipotentPrincipal.java | 4 +++ .../polaris/options/TargetPolarisOptions.java | 4 +++ 21 files changed, 115 insertions(+), 38 deletions(-) rename polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/{ETagService.java => ETagManager.java} (98%) rename polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/{NotModifiedException.java => MetadataNotModifiedException.java} (69%) rename polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/{NoOpETagService.java => NoOpETagManager.java} (95%) rename polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/{CsvETagService.java => CsvETagManager.java} (95%) diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md index 81e54a37..8b8a2d9b 100644 --- a/polaris-synchronizer/README.md +++ b/polaris-synchronizer/README.md @@ -16,16 +16,13 @@ Polaris specific entities, like principal-roles, catalog-roles, grants. failure. The tool can be scheduled on a cron to run periodic incremental syncs. The tool currently supports migrating the following Polaris Management entities: +* Optionally, Principals (with `--sync-principals` flag). Credentials will be different on the target instance. +* Optionally, assignment of Principal Roles to Principals (with `--sync-principals` flag) * Principal roles * Catalogs -* Catalog Roles -* Assignment of Catalog Roles to Principal Roles -* Grants - - -> :warning: Polaris principals and their assignments to Principal roles are not supported for migration -> by this tool. Migrating client credentials stored in Polaris is not possible nor is it secure. Polaris -> principals must be manually migrated between Polaris instances. + * Catalog Roles + * Assignment of Catalog Roles to Principal Roles + * Grants The tool currently supports migrating the following Iceberg entities: * Namespaces @@ -33,7 +30,7 @@ The tool currently supports migrating the following Iceberg entities: # Building the Tool from Source -**Prerequisite:** Must have Java installed in your machine (Java 21 is recommended and the minimum Java version) to use this CLI tool. +**Prerequisite:** Must have Java installed in your machine (Java 21 is recommended as the minimum Java version) to use this CLI tool. ``` gradlew build # build and run tests @@ -138,6 +135,11 @@ clientSecret = Running the synchronization requires minimal reconfiguration, can be run idempotently, and will attempt to only copy over the diff between the source and target Polaris instances. This can be achieved using the `sync-polaris` command. +> :warning: If you want to migrate principals and their assignments to principal-roles as well, run the tool with the +> `--sync-principals` flag. Please note that this will reset the client credentials for that principal on the target +> Polaris instance. The new credentials will be logged to stdout, ONLY for each newly created or overwritten principal. +> Please note that this output should be securely managed, client credentials should only ever be stored in a secure vault. + **Example** Running the synchronization between source Polaris instance using an access token, and a target Polaris instance using client credentials. ``` @@ -158,5 +160,6 @@ java -jar polaris-synchronizer-cli.jar sync-polaris \ ``` > :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source -> nor remove or modify them on the target. This is to accommodate that the tool itself will be running with the permission -> levels for these principals and roles, and we do not want to modify the tool's permissions at runtime. +> nor remove or modify them or their assignments to principals/principal-roles on the target. This is to accommodate that +> the tool itself will be running with the permission levels for these principals and roles, and we do not want to modify +> the tool's permissions at runtime. diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index 56b3e447..46370443 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -35,8 +35,8 @@ import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.access.AccessControlService; import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag; -import org.apache.polaris.tools.sync.polaris.catalog.ETagService; -import org.apache.polaris.tools.sync.polaris.catalog.NotModifiedException; +import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; +import org.apache.polaris.tools.sync.polaris.catalog.MetadataNotModifiedException; import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; @@ -69,7 +69,7 @@ public class PolarisSynchronizer { private final AccessControlService targetAccessControlService; - private final ETagService etagService; + private final ETagManager etagManager; public PolarisSynchronizer( Logger clientLogger, @@ -78,7 +78,7 @@ public PolarisSynchronizer( PrincipalWithCredentials targetOmnipotentPrincipal, PolarisService source, PolarisService target, - ETagService etagService) { + ETagManager etagManager) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.syncPlanner = synchronizationPlanner; @@ -95,7 +95,7 @@ public PolarisSynchronizer( this.targetOmnipotentPrincipalRole = targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal( targetOmnipotentPrincipal.getPrincipal().getName()); - this.etagService = etagService; + this.etagManager = etagManager; } /** @@ -200,6 +200,10 @@ public void syncPrincipals() { } } + /** + * Synchronize assigned principal roles for a principal from source to target. + * @param principalName the name of the principal to synchronize the principal roles for + */ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { List assignedPrincipalRolesSource; @@ -1208,7 +1212,7 @@ public void syncTables( } if (table instanceof BaseTableWithETag tableWithETag) { - etagService.storeETag(catalogName, tableId, tableWithETag.etag()); + etagManager.storeETag(catalogName, tableId, tableWithETag.etag()); } clientLogger.info( @@ -1235,7 +1239,7 @@ public void syncTables( Table table; if (sourceIcebergCatalog instanceof PolarisCatalog polarisCatalog) { - String etag = etagService.getETag(catalogName, tableId); + String etag = etagManager.getETag(catalogName, tableId); table = polarisCatalog.loadTable(tableId, etag); } else { table = sourceIcebergCatalog.loadTable(tableId); @@ -1250,7 +1254,7 @@ public void syncTables( } if (table instanceof BaseTableWithETag tableWithETag) { - etagService.storeETag(catalogName, tableId, tableWithETag.etag()); + etagManager.storeETag(catalogName, tableId, tableWithETag.etag()); } clientLogger.info( @@ -1260,7 +1264,7 @@ public void syncTables( catalogName, ++syncsCompleted, totalSyncsToComplete); - } catch (NotModifiedException e) { + } catch (MetadataNotModifiedException e) { clientLogger.info( "Table {} in namespace {} in catalog {} with was not modified, not overwriting in target catalog. - {}/{}", tableId, diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java index 58be6a95..fc0b978f 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java @@ -18,9 +18,19 @@ */ package org.apache.polaris.tools.sync.polaris.access; +/** + * Constants related to access control logic. + */ public class AccessControlConstants { + /** + * Property used to identify the omnipotent principal and all related access control entities + * across the source and target. + */ public static final String OMNIPOTENCE_PROPERTY = "IS_OMNIPOTENT_PRINCIPAL"; + /** + * Prefix to specify for naming an entity related to the omnipotent principal. + */ protected static final String OMNIPOTENT_PRINCIPAL_NAME_PREFIX = "omnipotent-principal-"; } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java index 1f28a031..c4bd4986 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java @@ -22,7 +22,12 @@ import org.apache.iceberg.TableOperations; import org.apache.iceberg.metrics.MetricsReporter; -/** Wrapper around {@link BaseTable} that contains the latest ETag for the table. */ +/** + * Wrapper around {@link BaseTable} that contains the latest ETag for the table. + * + * TODO: Remove this class once Iceberg gets first class support for ETags. + * in the canonical response types. +*/ public class BaseTableWithETag extends BaseTable { private final String etag; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java similarity index 98% rename from polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java rename to polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java index b06e4281..ded0a866 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java @@ -24,7 +24,7 @@ * Generic interface to provide and store ETags for tables within catalogs. This allows the storage * of the ETag to be completely independent from the tool. */ -public interface ETagService { +public interface ETagManager { /** * Retrieves the ETag for the table. diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java similarity index 69% rename from polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java rename to polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java index 6903c642..1ca85aad 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NotModifiedException.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java @@ -20,17 +20,23 @@ import org.apache.iceberg.catalog.TableIdentifier; -public class NotModifiedException extends RuntimeException { +/** + * Thrown when the metadata for a particular table was not modified from the version + * identified by a provided ETag. + * + * TODO: Remove once Iceberg has first class support for ETags. + */ +public class MetadataNotModifiedException extends RuntimeException { - public NotModifiedException(TableIdentifier tableIdentifier) { + public MetadataNotModifiedException(TableIdentifier tableIdentifier) { super("Table " + tableIdentifier + " was not modified."); } - public NotModifiedException(String message) { + public MetadataNotModifiedException(String message) { super(message); } - public NotModifiedException(String message, Throwable cause) { + public MetadataNotModifiedException(String message, Throwable cause) { super(message, cause); } } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java index 8a673c94..909cd8c6 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java @@ -27,6 +27,9 @@ /** * Wrapper table operations class that just allows fetching a provided table metadata. Used to build * a {@link org.apache.iceberg.BaseTable} without having to expose a full-fledged operations class. + * + * TODO: Remove this class once Iceberg gets first class support for ETags. + * in the canonical response types. */ public class MetadataWrapperTableOperations implements TableOperations { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java similarity index 95% rename from polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java rename to polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java index 04f3bd0c..ff8fa3d9 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java @@ -21,7 +21,7 @@ import org.apache.iceberg.catalog.TableIdentifier; /** Implementation that returns nothing and stores no ETags. */ -public class NoOpETagService implements ETagService { +public class NoOpETagManager implements ETagManager { @Override public String getETag(String catalogName, TableIdentifier tableIdentifier) { diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java index 553d7253..37b64e44 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java @@ -47,6 +47,9 @@ * Iceberg REST Api and build the table metadata. This is necessary since the existing {@link * RESTCatalog} does not provide a way to capture response headers to retrieve the ETag on a * loadTable request. + * + * TODO: Remove this class once Iceberg gets first class support for ETags. + * in the canonical response types. */ public class PolarisCatalog extends RESTCatalog implements Catalog, ViewCatalog, SupportsNamespaces, Configurable, Closeable { @@ -112,7 +115,7 @@ public Table loadTable(TableIdentifier ident) { * @param etag the etag * @return a {@link BaseTable} if no ETag was found in the response headers. A {@link * BaseTableWithETag} if an ETag was included in the response headers. - * @throws NotModifiedException if the Iceberg REST catalog responded with 304 NOT MODIFIED + * @throws MetadataNotModifiedException if the Iceberg REST catalog responded with 304 NOT MODIFIED */ public Table loadTable(TableIdentifier ident, String etag) { String catalogName = this.properties.get("warehouse"); @@ -143,7 +146,7 @@ public Table loadTable(TableIdentifier ident, String etag) { // api responded with 304 not modified, throw from here to signal if (response.statusCode() == HttpStatus.SC_NOT_MODIFIED) { - throw new NotModifiedException(ident); + throw new MetadataNotModifiedException(ident); } String body = response.body(); diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java index 82da075f..0747ce60 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java @@ -29,6 +29,9 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +/** + * No-op implementation that generates plans that direct no modifications to the target. + */ public class NoOpSyncPlanner implements SynchronizationPlanner { @Override diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java index 596dc4e7..a1b0a303 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java @@ -36,6 +36,10 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; +/** + * Command that creates the omnipotent principal to access entities internal to a catalog with the appropriate + * grants. + */ @CommandLine.Command( name = "create-omnipotent-principal", mixinStandardHelpOptions = true, diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java similarity index 95% rename from polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java rename to polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java index e391214c..bd477ac1 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagService.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java @@ -32,10 +32,10 @@ import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.tools.sync.polaris.catalog.ETagService; +import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; /** Implementation that stores/loads ETags to/from a CSV file. */ -public class CsvETagService implements ETagService, Closeable { +public class CsvETagManager implements ETagManager, Closeable { private static final String CATALOG_HEADER = "Catalog"; @@ -49,7 +49,7 @@ public class CsvETagService implements ETagService, Closeable { private final Map> tablesByCatalogName; - public CsvETagService(File file) throws IOException { + public CsvETagManager(File file) throws IOException { this.tablesByCatalogName = new HashMap<>(); this.file = file; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java index 08c36fc7..9cb3a57e 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java @@ -20,6 +20,9 @@ import picocli.CommandLine; +/** + * Main entrypoint into the tool CLI. Sets up some base level configuration for the commands to share. + */ @CommandLine.Command( name = "polaris-synchronizer", mixinStandardHelpOptions = true, @@ -33,7 +36,7 @@ public static void main(String[] args) { new CommandLine(new PolarisSynchronizerCLI()) .setExecutionExceptionHandler( (ex, cmd, parseResult) -> { - cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex)); + cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex)); // ensure stacktrace is printed return 1; }); commandLine.setUsageHelpWidth(150); diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index c4f0e043..768c9ee9 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.util.concurrent.Callable; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.tools.sync.polaris.catalog.ETagService; -import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagService; +import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; +import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager; import org.apache.polaris.tools.sync.polaris.options.SourceOmniPotentPrincipalOptions; import org.apache.polaris.tools.sync.polaris.options.SourcePolarisOptions; import org.apache.polaris.tools.sync.polaris.options.TargetOmnipotentPrincipal; @@ -37,6 +37,9 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; +/** + * Command to run the synchronization between a source and target Polaris instance. + */ @CommandLine.Command( name = "sync-polaris", mixinStandardHelpOptions = true, @@ -101,13 +104,13 @@ public Integer call() throws Exception { PrincipalWithCredentials targetOmniPotentPrincipal = targetOmniPotentPrincipalOptions.buildPrincipalWithCredentials(); - ETagService etagService; + ETagManager etagService; if (etagFilePath != null) { File etagFile = new File(etagFilePath); - etagService = new CsvETagService(etagFile); + etagService = new CsvETagManager(etagFile); } else { - etagService = new NoOpETagService(); + etagService = new NoOpETagManager(); } Runtime.getRuntime() diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java index c0107c16..87766d57 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java @@ -22,6 +22,11 @@ import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; +/** + * Base options class to define the common set of omnipotent principal authentication and connection properties. + * Can be used to give options different names based on the command while still ensuring they all + * satisfy the same sets of necessary properties. + */ public abstract class BaseOmnipotentPrincipalOptions { protected static final String PRINCIPAL_NAME = "omni-principal-name"; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java index 118489f7..19183b36 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java @@ -22,6 +22,11 @@ import org.apache.polaris.tools.sync.polaris.PolarisService; import org.apache.polaris.tools.sync.polaris.PolarisServiceFactory; +/** + * Base options class to define the common set of Polaris service admin authentication and connection properties. + * Can be used to give options different names based on the command while still ensuring they all + * satisfy the same sets of necessary properties. + */ public abstract class BasePolarisOptions { protected static final String BASE_URL = "base-url"; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java index 6a10e4db..571aac6a 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java @@ -20,6 +20,10 @@ import picocli.CommandLine; +/** + * Options for a generic Polaris instance. Use this for commands that only have to + * access one Polaris instance. + */ public class PolarisOptions extends BasePolarisOptions { @Override diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java index 90b7b46e..28c1ee24 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java @@ -20,6 +20,10 @@ import picocli.CommandLine; +/** + * Prefixes omnipotent principal option names with "source" tags to identify that these are + * the connection properties for the source instance. + */ public class SourceOmniPotentPrincipalOptions extends BaseOmnipotentPrincipalOptions { @CommandLine.Option( diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java index 7a811bc7..85f8b2a9 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java @@ -20,6 +20,10 @@ import picocli.CommandLine; +/** + * Prefixes service_admin connection option names with "source" tags to identify that these are + * the connection properties for the source instance. + */ public class SourcePolarisOptions extends BasePolarisOptions { @Override diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java index ac03b616..7b5e9c01 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java @@ -20,6 +20,10 @@ import picocli.CommandLine; +/** + * Prefixes omnipotent principal option names with "target" tags to identify that these are + * the connection properties for the target instance. + */ public class TargetOmnipotentPrincipal extends BaseOmnipotentPrincipalOptions { @CommandLine.Option( diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java index 6d6b2042..8e011758 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java @@ -20,6 +20,10 @@ import picocli.CommandLine; +/** + * Prefixes service_admin connection option names with "target" tags to identify that these are + * the connection properties for the target instance. + */ public class TargetPolarisOptions extends BasePolarisOptions { @Override From 05a280371a2a3d4bed4ee4dba17694b7af4a41f4 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Sun, 13 Apr 2025 15:31:22 -0700 Subject: [PATCH 06/18] Updated tests --- .../AccessControlAwarePlannerTest.java | 24 ++++++++ .../polaris/ModificationAwarePlannerTest.java | 50 +++++++++++++++ ...ourceParitySynchronizationPlannerTest.java | 61 ++++++++++++++----- 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java index 51d459ad..6f6cbc76 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java @@ -70,6 +70,30 @@ public void filtersRootPrincipal() { Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(rootPrincipalTarget)); } + @Test + public void filtersPrincipalAssignmentToOmnipotentPrincipalRole() { + SynchronizationPlanner accessControlAwarePlanner + = new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = accessControlAwarePlanner.planAssignPrincipalsToPrincipalRolesSync( + "principal", List.of(omnipotentPrincipalRoleSource), List.of(omnipotentPrincipalRoleTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget)); + } + + @Test + public void filtersAssignmentToServiceAdmin() { + SynchronizationPlanner accessControlAwarePlanner + = new AccessControlAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = accessControlAwarePlanner.planAssignPrincipalsToPrincipalRolesSync( + "principal", List.of(serviceAdminSource), List.of(serviceAdminTarget)); + + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource)); + Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget)); + } + private static final PrincipalRole omnipotentPrincipalRoleSource = new PrincipalRole() .name("omnipotent-principal-XXXXX") diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java index 3e6b1072..5be1788d 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java @@ -22,6 +22,8 @@ import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogPrivilege; import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.ExternalCatalog; @@ -88,6 +90,28 @@ public void testPrincipalNotModifiedWithResetClientId() { Assertions.assertTrue(plan.entitiesNotModified().contains(principalWithClientId)); } + private static final PrincipalRole assignedToPrincipal = new PrincipalRole().name("assigned"); + + @Test + public void principalRoleAlreadyAssignedToPrincipal() { + SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = modificationPlanner.planAssignPrincipalsToPrincipalRolesSync( + "principal", List.of(assignedToPrincipal), List.of(assignedToPrincipal)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(assignedToPrincipal)); + } + + @Test + public void principalRoleNotAssignedToPrincipal() { + SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = modificationPlanner.planAssignPrincipalsToPrincipalRolesSync( + "principal", List.of(assignedToPrincipal), List.of()); + + Assertions.assertFalse(plan.entitiesNotModified().contains(assignedToPrincipal)); + } + private static final PrincipalRole principalRole = new PrincipalRole().name("principal-role"); private static final PrincipalRole modifiedPrincipalRole = @@ -348,4 +372,30 @@ public void testOnlyGcsServiceAccountChangeGCP() { Assertions.assertTrue(plan.entitiesNotModified().contains(gcpCatalogGcsServiceAccountChange)); } + + private final static PrincipalRole assignedToCatalogRole = new PrincipalRole().name("assigned"); + + @Test + public void principalRoleAlreadyAssignedToCatalogRole() { + SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = modificationPlanner.planAssignPrincipalRolesToCatalogRolesSync( + "catalog", + "catalog-role", + List.of(assignedToCatalogRole), List.of(assignedToCatalogRole)); + + Assertions.assertTrue(plan.entitiesNotModified().contains(assignedToPrincipal)); + } + + @Test + public void principalRoleNotAssignedToCatalogRole() { + SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner()); + + SynchronizationPlan plan = modificationPlanner.planAssignPrincipalRolesToCatalogRolesSync( + "catalog", + "catalog-role", + List.of(assignedToCatalogRole), List.of()); + + Assertions.assertFalse(plan.entitiesNotModified().contains(assignedToPrincipal)); + } } diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java index e20e9365..18a961e7 100644 --- a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java +++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java @@ -89,6 +89,39 @@ public void testCreatesNewPrincipalOverwritesOldPrincipalRemovesDroppedPrincipal Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_3)); } + private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_1 = + new PrincipalRole().name("principal-role-1"); + + private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_2 = + new PrincipalRole().name("principal-role-2"); + + private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_3 = + new PrincipalRole().name("principal-role-3"); + + @Test + public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRoleForPrincipal() { + SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner(); + + SynchronizationPlan plan = + planner.planAssignPrincipalsToPrincipalRolesSync( + "principal", + List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2), + List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3)); + + Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1)); + + // special case: no concept of overwriting the assignment of a principal role + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2)); + + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3)); + } + private static final PrincipalRole PRINCIPAL_ROLE_1 = new PrincipalRole().name("principal-role-1"); @@ -180,13 +213,13 @@ public void testCreatesNewGrantResourceRemovesDroppedGrantResource() { Assertions.assertTrue(plan.entitiesToRemove().contains(GRANT_3)); } - private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_1 = + private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_1 = new PrincipalRole().name("principal-role-1"); - private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_2 = + private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_2 = new PrincipalRole().name("principal-role-2"); - private static final PrincipalRole ASSIGNED_PRINCIPAL_ROLE_3 = + private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_3 = new PrincipalRole().name("principal-role-3"); @Test @@ -197,21 +230,21 @@ public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRole() { planner.planAssignPrincipalRolesToCatalogRolesSync( "catalog", "catalogRole", - List.of(ASSIGNED_PRINCIPAL_ROLE_1, ASSIGNED_PRINCIPAL_ROLE_2), - List.of(ASSIGNED_PRINCIPAL_ROLE_2, ASSIGNED_PRINCIPAL_ROLE_3)); + List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2), + List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3)); - Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_1)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_1)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_1)); + Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1)); // special case: no concept of overwriting the assignment of a principal role - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_2)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_2)); - Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_2)); + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2)); + Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2)); - Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_PRINCIPAL_ROLE_3)); - Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_PRINCIPAL_ROLE_3)); - Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_PRINCIPAL_ROLE_3)); + Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3)); + Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3)); + Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3)); } private static final Namespace NS_1 = Namespace.of("ns1"); From a0d024d5b84a99dc8fa40fb6bccd5e45af2c0c5b Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Mon, 14 Apr 2025 19:06:46 -0700 Subject: [PATCH 07/18] Add generic Polaris entity source and target- not tied to API --- .../tools/sync/polaris/PolarisService.java | 375 ----------------- .../sync/polaris/PolarisServiceFactory.java | 91 ---- .../sync/polaris/PolarisSynchronizer.java | 389 +++++++----------- .../polaris/access/AccessControlService.java | 26 +- .../service/IcebergCatalogService.java | 35 ++ .../sync/polaris/service/PolarisService.java | 67 +++ .../service/impl/PolarisApiService.java | 256 ++++++++++++ .../impl/PolarisIcebergCatalogService.java | 131 ++++++ .../CreateOmnipotentPrincipalCommand.java | 27 +- .../sync/polaris/PolarisServiceFactory.java | 40 ++ .../sync/polaris/SyncPolarisCommand.java | 75 ++-- .../BaseOmnipotentPrincipalOptions.java | 58 --- .../polaris/options/BasePolarisOptions.java | 77 ---- .../sync/polaris/options/PolarisOptions.java | 94 ----- .../SourceOmniPotentPrincipalOptions.java | 55 --- .../polaris/options/SourcePolarisOptions.java | 94 ----- .../options/TargetOmnipotentPrincipal.java | 55 --- .../polaris/options/TargetPolarisOptions.java | 94 ----- 18 files changed, 755 insertions(+), 1284 deletions(-) delete mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java delete mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java create mode 100644 polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java delete mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java deleted file mode 100644 index 03906bd4..00000000 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisService.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.http.HttpStatus; -import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.polaris.core.admin.model.AddGrantRequest; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRequest; -import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.GrantResource; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.RevokeGrantRequest; -import org.apache.polaris.management.ApiException; -import org.apache.polaris.management.client.PolarisManagementDefaultApi; -import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; - -/** - * Service class that wraps Polaris HTTP client and performs recursive operations like drops on - * overwrites. - */ -public class PolarisService { - - private final PolarisManagementDefaultApi api; - - private final Map catalogProperties; - - public PolarisService(PolarisManagementDefaultApi api, Map catalogProperties) { - this.api = api; - this.catalogProperties = catalogProperties; - } - - public List listPrincipals() { - return this.api.listPrincipals().getPrincipals(); - } - - public Principal getPrincipal(String principalName) { - return this.api.getPrincipal(principalName); - } - - public boolean principalExists(String principalName) { - try { - getPrincipal(principalName); - return true; - } catch (ApiException apiException) { - if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { - return false; - } - throw apiException; - } - } - - public PrincipalWithCredentials createPrincipal(Principal principal, boolean overwrite) { - if (overwrite) { - removePrincipal(principal.getName()); - } - - CreatePrincipalRequest request = new CreatePrincipalRequest().principal(principal); - return this.api.createPrincipal(request); - } - - public void removePrincipal(String principalName) { - this.api.deletePrincipal(principalName); - } - - public void assignPrincipalRoleToPrincipal(String principalName, String principalRoleName) { - GrantPrincipalRoleRequest request = - new GrantPrincipalRoleRequest().principalRole(new PrincipalRole().name(principalRoleName)); - this.api.assignPrincipalRole(principalName, request); - } - - public void revokePrincipalRoleFromPrincipal(String principalName, String principalRoleName) { - this.api.revokePrincipalRole(principalName, principalRoleName); - } - - public void createPrincipalRole(PrincipalRole principalRole, boolean overwrite) { - if (overwrite) { - removePrincipalRole(principalRole.getName()); - } - CreatePrincipalRoleRequest request = - new CreatePrincipalRoleRequest().principalRole(principalRole); - this.api.createPrincipalRole(request); - } - - public List listPrincipalRolesAssignedForPrincipal(String principalName) { - return this.api.listPrincipalRolesAssigned(principalName).getRoles(); - } - - public List listPrincipalRoles() { - return this.api.listPrincipalRoles().getRoles(); - } - - public List listAssigneePrincipalRolesForCatalogRole( - String catalogName, String catalogRoleName) { - return this.api - .listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName) - .getRoles(); - } - - public void assignCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, String catalogRoleName) { - GrantCatalogRoleRequest request = - new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName)); - this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request); - } - - public void removeCatalogRoleFromPrincipalRole( - String principalRoleName, String catalogName, String catalogRoleName) { - this.api.revokeCatalogRoleFromPrincipalRole(principalRoleName, catalogName, catalogRoleName); - } - - public PrincipalRole getPrincipalRole(String principalRoleName) { - return this.api.getPrincipalRole(principalRoleName); - } - - public boolean principalRoleExists(String principalRoleName) { - try { - getPrincipalRole(principalRoleName); - return true; - } catch (ApiException apiException) { - if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { - return false; - } - throw apiException; - } - } - - public void removePrincipalRole(String principalRoleName) { - this.api.deletePrincipalRole(principalRoleName); - } - - public List listCatalogs() { - return this.api.listCatalogs().getCatalogs(); - } - - public void createCatalog(Catalog catalog) { - CreateCatalogRequest request = new CreateCatalogRequest().catalog(catalog); - this.api.createCatalog(request); - } - - /** - * Performs a cascading drop on the catalog before recreating. - * - * @param catalog - * @param omnipotentPrincipal necessary to initialize an Iceberg catalog to drop catalog internals - */ - public void overwriteCatalog(Catalog catalog, PrincipalWithCredentials omnipotentPrincipal) { - removeCatalogCascade(catalog.getName(), omnipotentPrincipal); - createCatalog(catalog); - } - - /** - * Recursively discover all namespaces contained within an Iceberg catalog. - * - * @param catalog - * @return a list of all the namespaces in the catalog - */ - private List discoverAllNamespaces(org.apache.iceberg.catalog.Catalog catalog) { - List namespaces = new ArrayList<>(); - namespaces.add(Namespace.empty()); - - if (catalog instanceof SupportsNamespaces namespaceCatalog) { - namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, Namespace.empty())); - } - - return namespaces; - } - - /** - * Discover all child namespaces of a given namespace. - * - * @param namespaceCatalog a catalog that supports nested namespaces - * @param namespace the namespace to look under - * @return a list of all child namespaces - */ - private List discoverContainedNamespaces( - SupportsNamespaces namespaceCatalog, Namespace namespace) { - List immediateChildren = namespaceCatalog.listNamespaces(namespace); - - List namespaces = new ArrayList<>(); - - for (Namespace ns : immediateChildren) { - namespaces.add(ns); - - // discover children of child namespace - namespaces.addAll(discoverContainedNamespaces(namespaceCatalog, ns)); - } - - return namespaces; - } - - /** - * Perform a cascading drop of a catalog. Removes all namespaces, tables, catalog-roles first. - * - * @param catalogName - * @param omnipotentPrincipal - */ - public void removeCatalogCascade( - String catalogName, PrincipalWithCredentials omnipotentPrincipal) { - org.apache.iceberg.catalog.Catalog icebergCatalog = - initializeCatalog(catalogName, omnipotentPrincipal); - - // find all namespaces in the catalog - List namespaces = discoverAllNamespaces(icebergCatalog); - - List tables = new ArrayList<>(); - - // find all tables in the catalog - for (Namespace ns : namespaces) { - if (!ns.isEmpty()) { - tables.addAll(icebergCatalog.listTables(ns)); - } - } - - // drop every table in the catalog - for (TableIdentifier table : tables) { - icebergCatalog.dropTable(table); - } - - // drop every namespace in the catalog, note that because we discovered the namespaces - // parent-first, we should reverse over the namespaces to ensure that we drop child namespaces - // before we drop parent namespaces, as we cannot drop nonempty namespaces - for (Namespace ns : namespaces.reversed()) { - // NOTE: this is checking if the namespace is not the empty namespace, not if it is empty - // in the sense of containing no tables/namespaces - if (!ns.isEmpty() && icebergCatalog instanceof SupportsNamespaces namespaceCatalog) { - namespaceCatalog.dropNamespace(ns); - } - } - - List catalogRoles = listCatalogRoles(catalogName); - - // remove catalog roles under catalog - for (CatalogRole catalogRole : catalogRoles) { - if (catalogRole.getName().equals("catalog_admin")) continue; - - removeCatalogRole(catalogName, catalogRole.getName()); - } - - this.api.deleteCatalog(catalogName); - } - - public List listCatalogRoles(String catalogName) { - return this.api.listCatalogRoles(catalogName).getRoles(); - } - - public CatalogRole getCatalogRole(String catalogName, String catalogRoleName) { - return this.api.getCatalogRole(catalogName, catalogRoleName); - } - - public boolean catalogRoleExists(String catalogName, String catalogRoleName) { - try { - getCatalogRole(catalogName, catalogRoleName); - return true; - } catch (ApiException apiException) { - if (apiException.getCode() == HttpStatus.SC_NOT_FOUND) { - return false; - } - throw apiException; - } - } - - public void assignCatalogRole( - String principalRoleName, String catalogName, String catalogRoleName) { - GrantCatalogRoleRequest request = - new GrantCatalogRoleRequest().catalogRole(new CatalogRole().name(catalogRoleName)); - this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request); - } - - public void createCatalogRole(String catalogName, CatalogRole catalogRole, boolean overwrite) { - if (overwrite) { - removeCatalogRole(catalogName, catalogRole.getName()); - } - - CreateCatalogRoleRequest request = new CreateCatalogRoleRequest().catalogRole(catalogRole); - this.api.createCatalogRole(catalogName, request); - } - - public void removeCatalogRole(String catalogName, String catalogRoleName) { - this.api.deleteCatalogRole(catalogName, catalogRoleName); - } - - public List listGrants(String catalogName, String catalogRoleName) { - return this.api.listGrantsForCatalogRole(catalogName, catalogRoleName).getGrants(); - } - - public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { - AddGrantRequest addGrantRequest = new AddGrantRequest().grant(grant); - this.api.addGrantToCatalogRole(catalogName, catalogRoleName, addGrantRequest); - } - - public void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant) { - RevokeGrantRequest revokeGrantRequest = new RevokeGrantRequest().grant(grant); - this.api.revokeGrantFromCatalogRole(catalogName, catalogRoleName, false, revokeGrantRequest); - } - - public org.apache.iceberg.catalog.Catalog initializeCatalog( - String catalogName, PrincipalWithCredentials migratorPrincipal) { - Map currentCatalogProperties = new HashMap<>(catalogProperties); - currentCatalogProperties.put("warehouse", catalogName); - - String clientId = migratorPrincipal.getCredentials().getClientId(); - String clientSecret = migratorPrincipal.getCredentials().getClientSecret(); - currentCatalogProperties.putIfAbsent( - "credential", String.format("%s:%s", clientId, clientSecret)); - currentCatalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL"); - - return CatalogUtil.loadCatalog( - PolarisCatalog.class.getName(), "SOURCE_CATALOG_REST", currentCatalogProperties, null); - } - - /** - * Perform cascading drop of a namespace. - * - * @param icebergCatalog the iceberg catalog to use - * @param namespace the namespace to drop - */ - public void dropNamespaceCascade( - org.apache.iceberg.catalog.Catalog icebergCatalog, Namespace namespace) { - if (icebergCatalog instanceof SupportsNamespaces namespaceCatalog) { - List namespaces = discoverContainedNamespaces(namespaceCatalog, namespace); - - List tables = new ArrayList<>(); - - for (Namespace ns : namespaces) { - tables.addAll(icebergCatalog.listTables(ns)); - } - - tables.addAll(icebergCatalog.listTables(namespace)); - - for (TableIdentifier table : tables) { - icebergCatalog.dropTable(table); - } - - // go over in reverse order of namespaces since we discover namespaces - // in the parent -> child order, so we need to drop all children - // before we can drop the parent - for (Namespace ns : namespaces.reversed()) { - namespaceCatalog.dropNamespace(ns); - } - - namespaceCatalog.dropNamespace(namespace); - } - } -} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java deleted file mode 100644 index 72d0f78c..00000000 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import org.apache.http.HttpHeaders; -import org.apache.polaris.management.ApiClient; -import org.apache.polaris.management.client.PolarisManagementDefaultApi; -import org.apache.polaris.tools.sync.polaris.http.OAuth2Util; - -/** Used to initialize a {@link PolarisService}. */ -public class PolarisServiceFactory { - - private static void validatePolarisInstanceProperties( - String baseUrl, - String accessToken, - String oauth2ServerUri, - String clientId, - String clientSecret, - String scope) { - if (baseUrl == null) { - throw new IllegalArgumentException("baseUrl is required but was not provided"); - } - - if (accessToken != null) { - return; - } - - final String oauthErrorMessage = - "Either the accessToken property must be provided, or all of oauth2ServerUri, clientId, clientSecret, scope"; - - if (oauth2ServerUri == null || clientId == null || clientSecret == null || scope == null) { - throw new IllegalArgumentException(oauthErrorMessage); - } - } - - public static PolarisService newPolarisService( - String baseUrl, String oauth2ServerUri, String clientId, String clientSecret, String scope) - throws IOException { - validatePolarisInstanceProperties( - baseUrl, null /* accessToken */, oauth2ServerUri, clientId, clientSecret, scope); - - String accessToken = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope); - - return newPolarisService(baseUrl, accessToken); - } - - public static PolarisService newPolarisService(String baseUrl, String accessToken) { - validatePolarisInstanceProperties( - baseUrl, - accessToken, - null, /* oauth2ServerUri */ - null, /* clientId */ - null, /* clientSecret */ - null /* scope */ - ); - - ApiClient client = new ApiClient(); - client.updateBaseUri(baseUrl + "/api/management/v1"); - - // TODO: Add token refresh - client.setRequestInterceptor( - requestBuilder -> { - requestBuilder.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); - }); - - Map catalogProperties = new HashMap<>(); - catalogProperties.putIfAbsent("uri", baseUrl + "/api/catalog"); - - PolarisManagementDefaultApi polarisClient = new PolarisManagementDefaultApi(client); - return new PolarisService(polarisClient, catalogProperties); - } -} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index 46370443..014ccf71 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -25,7 +25,6 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogRole; @@ -33,13 +32,14 @@ import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.tools.sync.polaris.access.AccessControlService; import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag; import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; import org.apache.polaris.tools.sync.polaris.catalog.MetadataNotModifiedException; -import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan; +import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService; +import org.apache.polaris.tools.sync.polaris.service.PolarisService; +import org.apache.polaris.tools.sync.polaris.service.impl.PolarisIcebergCatalogService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,44 +57,19 @@ public class PolarisSynchronizer { private final PolarisService target; - private final PrincipalWithCredentials sourceOmnipotentPrincipal; - - private final PrincipalWithCredentials targetOmnipotentPrincipal; - - private final PrincipalRole sourceOmnipotentPrincipalRole; - - private final PrincipalRole targetOmnipotentPrincipalRole; - - private final AccessControlService sourceAccessControlService; - - private final AccessControlService targetAccessControlService; - private final ETagManager etagManager; public PolarisSynchronizer( Logger clientLogger, SynchronizationPlanner synchronizationPlanner, - PrincipalWithCredentials sourceOmnipotentPrincipal, - PrincipalWithCredentials targetOmnipotentPrincipal, PolarisService source, PolarisService target, ETagManager etagManager) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; this.syncPlanner = synchronizationPlanner; - this.sourceOmnipotentPrincipal = sourceOmnipotentPrincipal; - this.targetOmnipotentPrincipal = targetOmnipotentPrincipal; this.source = source; this.target = target; - this.sourceAccessControlService = new AccessControlService(source); - this.targetAccessControlService = new AccessControlService(target); - - this.sourceOmnipotentPrincipalRole = - sourceAccessControlService.getOmnipotentPrincipalRoleForPrincipal( - sourceOmnipotentPrincipal.getPrincipal().getName()); - this.targetOmnipotentPrincipalRole = - targetAccessControlService.getOmnipotentPrincipalRoleForPrincipal( - targetOmnipotentPrincipal.getPrincipal().getName()); this.etagManager = etagManager; } @@ -154,7 +129,7 @@ public void syncPrincipals() { for (Principal principal : principalSyncPlan.entitiesToCreate()) { try { - PrincipalWithCredentials createdPrincipal = target.createPrincipal(principal, false /* overwrite */); + PrincipalWithCredentials createdPrincipal = target.createPrincipal(principal); clientLogger.info("Created principal {} on target. Target credentials: {}:{} - {}/{}", principal.getName(), createdPrincipal.getCredentials().getClientId(), @@ -170,7 +145,8 @@ public void syncPrincipals() { for (Principal principal : principalSyncPlan.entitiesToOverwrite()) { try { - PrincipalWithCredentials overwrittenPrincipal = target.createPrincipal(principal, true /* overwrite */); + target.dropPrincipal(principal.getName()); + PrincipalWithCredentials overwrittenPrincipal = target.createPrincipal(principal); clientLogger.info("Overwrote principal {} on target. Target credentials: {}:{} - {}/{}", principal.getName(), overwrittenPrincipal.getCredentials().getClientId(), @@ -186,7 +162,7 @@ public void syncPrincipals() { for (Principal principal : principalSyncPlan.entitiesToRemove()) { try { - target.removePrincipal(principal.getName()); + target.dropPrincipal(principal.getName()); clientLogger.info("Removed principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { @@ -208,7 +184,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { List assignedPrincipalRolesSource; try { - assignedPrincipalRolesSource = source.listPrincipalRolesAssignedForPrincipal(principalName); + assignedPrincipalRolesSource = source.listPrincipalRolesAssigned(principalName); clientLogger.info("Listed {} assigned principal-roles for principal {} from source.", assignedPrincipalRolesSource.size(), principalName); } catch (Exception e) { @@ -219,7 +195,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { List assignedPrincipalRolesTarget; try { - assignedPrincipalRolesTarget = target.listPrincipalRolesAssignedForPrincipal(principalName); + assignedPrincipalRolesTarget = target.listPrincipalRolesAssigned(principalName); clientLogger.info("Listed {} assigned principal-roles for principal {} from target.", assignedPrincipalRolesTarget.size(), principalName); } catch (Exception e) { @@ -251,7 +227,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) { try { - target.assignPrincipalRoleToPrincipal(principalName, principalRole.getName()); + target.assignPrincipalRole(principalName, principalRole.getName()); clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { @@ -262,7 +238,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) { try { - target.assignPrincipalRoleToPrincipal(principalName, principalRole.getName()); + target.assignPrincipalRole(principalName, principalRole.getName()); clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { @@ -273,7 +249,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) { try { - target.revokePrincipalRoleFromPrincipal(principalName, principalRole.getName()); + target.revokePrincipalRole(principalName, principalRole.getName()); clientLogger.info("Revoked principal-role {} from principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { @@ -327,7 +303,7 @@ public void syncPrincipalRoles() { for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToCreate()) { try { - target.createPrincipalRole(principalRole, false); + target.createPrincipalRole(principalRole); clientLogger.info( "Created principal-role {} on target. - {}/{}", principalRole.getName(), @@ -345,7 +321,8 @@ public void syncPrincipalRoles() { for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToOverwrite()) { try { - target.createPrincipalRole(principalRole, true); + target.dropPrincipalRole(principalRole.getName()); + target.createPrincipalRole(principalRole); clientLogger.info( "Overwrote principal-role {} on target. - {}/{}", principalRole.getName(), @@ -363,7 +340,7 @@ public void syncPrincipalRoles() { for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToRemove()) { try { - target.removePrincipalRole(principalRole.getName()); + target.dropPrincipalRole(principalRole.getName()); clientLogger.info( "Removed principal-role {} on target. - {}/{}", principalRole.getName(), @@ -454,7 +431,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) { try { - target.assignCatalogRoleToPrincipalRole( + target.assignCatalogRole( principalRole.getName(), catalogName, catalogRoleName); clientLogger.info( "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}", @@ -477,7 +454,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) { try { - target.assignCatalogRoleToPrincipalRole( + target.assignCatalogRole( principalRole.getName(), catalogName, catalogRoleName); clientLogger.info( "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}", @@ -500,7 +477,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) { try { - target.removeCatalogRoleFromPrincipalRole( + target.revokeCatalogRole( principalRole.getName(), catalogName, catalogRoleName); clientLogger.info( "Revoked principal-role {} from catalog-role {} in catalog {}. - {}/{}", @@ -588,8 +565,8 @@ public void syncCatalogs() { for (Catalog catalog : catalogSyncPlan.entitiesToOverwrite()) { try { - setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName()); - target.overwriteCatalog(catalog, targetOmnipotentPrincipal); + target.dropCatalogCascade(catalog.getName()); + target.createCatalog(catalog); clientLogger.info( "Overwrote catalog {}. - {}/{}", catalog.getName(), @@ -607,8 +584,7 @@ public void syncCatalogs() { for (Catalog catalog : catalogSyncPlan.entitiesToRemove()) { try { - setupOmnipotentCatalogRoleIfNotExistsTarget(catalog.getName()); - target.removeCatalogCascade(catalog.getName(), targetOmnipotentPrincipal); + target.dropCatalogCascade(catalog.getName()); clientLogger.info( "Removed catalog {}. - {}/{}", catalog.getName(), @@ -627,10 +603,10 @@ public void syncCatalogs() { for (Catalog catalog : catalogSyncPlan.entitiesToSyncChildren()) { syncCatalogRoles(catalog.getName()); - org.apache.iceberg.catalog.Catalog sourceIcebergCatalog; + IcebergCatalogService sourceIcebergCatalogService; try { - sourceIcebergCatalog = initializeIcebergCatalogSource(catalog.getName()); + sourceIcebergCatalogService = source.initializeIcebergCatalogService(catalog.getName()); clientLogger.info( "Initialized Iceberg REST catalog for Polaris catalog {} on source.", catalog.getName()); @@ -642,10 +618,10 @@ public void syncCatalogs() { continue; } - org.apache.iceberg.catalog.Catalog targetIcebergCatalog; + IcebergCatalogService targetIcebergCatalogService; try { - targetIcebergCatalog = initializeIcebergCatalogTarget(catalog.getName()); + targetIcebergCatalogService = target.initializeIcebergCatalogService(catalog.getName()); clientLogger.info( "Initialized Iceberg REST catalog for Polaris catalog {} on target.", catalog.getName()); @@ -658,7 +634,7 @@ public void syncCatalogs() { } syncNamespaces( - catalog.getName(), Namespace.empty(), sourceIcebergCatalog, targetIcebergCatalog); + catalog.getName(), Namespace.empty(), sourceIcebergCatalogService, targetIcebergCatalogService); } } @@ -729,7 +705,7 @@ public void syncCatalogRoles(String catalogName) { for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToCreate()) { try { - target.createCatalogRole(catalogName, catalogRole, false); + target.createCatalogRole(catalogName, catalogRole); clientLogger.info( "Created catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -749,7 +725,8 @@ public void syncCatalogRoles(String catalogName) { for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToOverwrite()) { try { - target.createCatalogRole(catalogName, catalogRole, true); + target.dropCatalogRole(catalogName, catalogRole.getName()); + target.createCatalogRole(catalogName, catalogRole); clientLogger.info( "Overwrote catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -769,7 +746,7 @@ public void syncCatalogRoles(String catalogName) { for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToRemove()) { try { - target.removeCatalogRole(catalogName, catalogRole.getName()); + target.dropCatalogRole(catalogName, catalogRole.getName()); clientLogger.info( "Removed catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -930,208 +907,158 @@ private void syncGrants(String catalogName, String catalogRoleName) { } /** - * Setup an omnipotent principal for the provided catalog on the target Polaris instance. + * Sync namespaces contained within a parent namespace. * * @param catalogName + * @param parentNamespace + * @param sourceIcebergCatalogService + * @param targetIcebergCatalogService */ - private void setupOmnipotentCatalogRoleIfNotExistsTarget(String catalogName) { - if (!this.targetAccessControlService.omnipotentCatalogRoleExists(catalogName)) { + public void syncNamespaces( + String catalogName, + Namespace parentNamespace, + IcebergCatalogService sourceIcebergCatalogService, + IcebergCatalogService targetIcebergCatalogService) { + List namespacesSource; + + try { + namespacesSource = sourceIcebergCatalogService.listNamespaces(parentNamespace); clientLogger.info( - "No omnipotent catalog-role exists for catalog {} on target. Going to set one up.", + "Listed {} namespaces in namespace {} for catalog {} from source.", + namespacesSource.size(), + parentNamespace, catalogName); - - targetAccessControlService.setupOmnipotentRoleForCatalog( - catalogName, targetOmnipotentPrincipalRole, false, true); - - clientLogger.info("Setup omnipotent catalog-role for catalog {} on target.", catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list namespaces in namespace {} for catalog {} from source.", + parentNamespace, + catalogName, + e); + return; } - } - /** - * Setup an omnipotent principal for the provided catalog on the target Polaris instance. - * - * @param catalogName - */ - private void setupOmnipotentCatalogRoleIfNotExistsSource(String catalogName) { - if (!this.sourceAccessControlService.omnipotentCatalogRoleExists(catalogName)) { + List namespacesTarget; + + try { + namespacesTarget = targetIcebergCatalogService.listNamespaces(parentNamespace); clientLogger.info( - "No omnipotent catalog-role exists for catalog {} on source. Going to set one up.", + "Listed {} namespaces in namespace {} for catalog {} from target.", + namespacesTarget.size(), + parentNamespace, catalogName); - - sourceAccessControlService.setupOmnipotentRoleForCatalog( - catalogName, sourceOmnipotentPrincipalRole, false, false); - - clientLogger.info("Setup omnipotent catalog-role for catalog {} on source.", catalogName); + } catch (Exception e) { + clientLogger.error( + "Failed to list namespaces in namespace {} for catalog {} from target.", + parentNamespace, + catalogName, + e); + return; } - } - public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogSource(String catalogName) { - setupOmnipotentCatalogRoleIfNotExistsSource(catalogName); - return source.initializeCatalog(catalogName, sourceOmnipotentPrincipal); - } + SynchronizationPlan namespaceSynchronizationPlan = + syncPlanner.planNamespaceSync( + catalogName, parentNamespace, namespacesSource, namespacesTarget); - public org.apache.iceberg.catalog.Catalog initializeIcebergCatalogTarget(String catalogName) { - setupOmnipotentCatalogRoleIfNotExistsTarget(catalogName); - return target.initializeCatalog(catalogName, targetOmnipotentPrincipal); - } + int syncsCompleted = 0; + int totalSyncsToComplete = totalSyncsToComplete(namespaceSynchronizationPlan); - /** - * Sync namespaces contained within a parent namespace. - * - * @param catalogName - * @param parentNamespace - * @param sourceIcebergCatalog - * @param targetIcebergCatalog - */ - public void syncNamespaces( - String catalogName, - Namespace parentNamespace, - org.apache.iceberg.catalog.Catalog sourceIcebergCatalog, - org.apache.iceberg.catalog.Catalog targetIcebergCatalog) { - // no namespaces to sync if catalog does not implement SupportsNamespaces - if (sourceIcebergCatalog instanceof SupportsNamespaces sourceNamespaceCatalog - && targetIcebergCatalog instanceof SupportsNamespaces targetNamespaceCatalog) { - List namespacesSource; + namespaceSynchronizationPlan + .entitiesNotModified() + .forEach( + namespace -> + clientLogger.info( + "No change detected for namespace {} in namespace {} for catalog {}, skipping.", + namespace, + parentNamespace, + catalogName)); + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToCreate()) { try { - namespacesSource = sourceNamespaceCatalog.listNamespaces(parentNamespace); + Map namespaceMetadata = sourceIcebergCatalogService.loadNamespaceMetadata(namespace); + targetIcebergCatalogService.createNamespace(namespace, namespaceMetadata); clientLogger.info( - "Listed {} namespaces in namespace {} for catalog {} from source.", - namespacesSource.size(), + "Created namespace {} in namespace {} for catalog {} - {}/{}", + namespace, parentNamespace, - catalogName); + catalogName, + ++syncsCompleted, + totalSyncsToComplete); } catch (Exception e) { clientLogger.error( - "Failed to list namespaces in namespace {} for catalog {} from source.", + "Failed to create namespace {} in namespace {} for catalog {} - {}/{}", + namespace, parentNamespace, catalogName, + ++syncsCompleted, + totalSyncsToComplete, e); - return; } + } - List namespacesTarget; - + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToOverwrite()) { try { - namespacesTarget = targetNamespaceCatalog.listNamespaces(parentNamespace); - clientLogger.info( - "Listed {} namespaces in namespace {} for catalog {} from target.", - namespacesTarget.size(), - parentNamespace, - catalogName); - } catch (Exception e) { - clientLogger.error( - "Failed to list namespaces in namespace {} for catalog {} from target.", - parentNamespace, - catalogName, - e); - return; - } + Map sourceNamespaceMetadata = + sourceIcebergCatalogService.loadNamespaceMetadata(namespace); + Map targetNamespaceMetadata = + targetIcebergCatalogService.loadNamespaceMetadata(namespace); - SynchronizationPlan namespaceSynchronizationPlan = - syncPlanner.planNamespaceSync( - catalogName, parentNamespace, namespacesSource, namespacesTarget); - - int syncsCompleted = 0; - int totalSyncsToComplete = totalSyncsToComplete(namespaceSynchronizationPlan); - - namespaceSynchronizationPlan - .entitiesNotModified() - .forEach( - namespace -> - clientLogger.info( - "No change detected for namespace {} in namespace {} for catalog {}, skipping.", - namespace, - parentNamespace, - catalogName)); - - for (Namespace namespace : namespaceSynchronizationPlan.entitiesToCreate()) { - try { - targetNamespaceCatalog.createNamespace(namespace); + if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) { clientLogger.info( - "Created namespace {} in namespace {} for catalog {} - {}/{}", + "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}", namespace, parentNamespace, catalogName, ++syncsCompleted, totalSyncsToComplete); - } catch (Exception e) { - clientLogger.error( - "Failed to create namespace {} in namespace {} for catalog {} - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete, - e); + continue; } - } - - for (Namespace namespace : namespaceSynchronizationPlan.entitiesToOverwrite()) { - try { - Map sourceNamespaceMetadata = - sourceNamespaceCatalog.loadNamespaceMetadata(namespace); - Map targetNamespaceMetadata = - targetNamespaceCatalog.loadNamespaceMetadata(namespace); - - if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) { - clientLogger.info( - "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete); - continue; - } - target.dropNamespaceCascade(targetIcebergCatalog, namespace); - targetNamespaceCatalog.createNamespace(namespace, sourceNamespaceMetadata); + targetIcebergCatalogService.setNamespaceProperties(namespace, sourceNamespaceMetadata); - clientLogger.info( - "Overwrote namespace {} in namespace {} for catalog {} - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete); - } catch (Exception e) { - clientLogger.error( - "Failed to overwrite namespace {} in namespace {} for catalog {} - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete, - e); - } + clientLogger.info( + "Overwrote namespace metadata {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to overwrite namespace metadata {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); } + } - for (Namespace namespace : namespaceSynchronizationPlan.entitiesToRemove()) { - try { - target.dropNamespaceCascade(targetIcebergCatalog, namespace); - clientLogger.info( - "Removed namespace {} in namespace {} for catalog {} - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete); - } catch (Exception e) { - clientLogger.error( - "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}", - namespace, - parentNamespace, - catalogName, - ++syncsCompleted, - totalSyncsToComplete, - e); - } + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToRemove()) { + try { + targetIcebergCatalogService.dropNamespaceCascade(namespace); + clientLogger.info( + "Removed namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete); + } catch (Exception e) { + clientLogger.error( + "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}", + namespace, + parentNamespace, + catalogName, + ++syncsCompleted, + totalSyncsToComplete, + e); } + } - for (Namespace namespace : namespaceSynchronizationPlan.entitiesToSyncChildren()) { - syncTables(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog); - syncNamespaces(catalogName, namespace, sourceIcebergCatalog, targetIcebergCatalog); - } + for (Namespace namespace : namespaceSynchronizationPlan.entitiesToSyncChildren()) { + syncTables(catalogName, namespace, sourceIcebergCatalogService, targetIcebergCatalogService); + syncNamespaces(catalogName, namespace, sourceIcebergCatalogService, targetIcebergCatalogService); } } @@ -1140,18 +1067,18 @@ public void syncNamespaces( * * @param catalogName * @param namespace - * @param sourceIcebergCatalog - * @param targetIcebergCatalog + * @param sourceIcebergCatalogService + * @param targetIcebergCatalogService */ public void syncTables( String catalogName, Namespace namespace, - org.apache.iceberg.catalog.Catalog sourceIcebergCatalog, - org.apache.iceberg.catalog.Catalog targetIcebergCatalog) { + IcebergCatalogService sourceIcebergCatalogService, + IcebergCatalogService targetIcebergCatalogService) { Set sourceTables; try { - sourceTables = new HashSet<>(sourceIcebergCatalog.listTables(namespace)); + sourceTables = new HashSet<>(sourceIcebergCatalogService.listTables(namespace)); clientLogger.info( "Listed {} tables in namespace {} for catalog {} on source.", sourceTables.size(), @@ -1169,7 +1096,7 @@ public void syncTables( Set targetTables; try { - targetTables = new HashSet<>(targetIcebergCatalog.listTables(namespace)); + targetTables = new HashSet<>(targetIcebergCatalogService.listTables(namespace)); clientLogger.info( "Listed {} tables in namespace {} for catalog {} on target.", targetTables.size(), @@ -1202,10 +1129,10 @@ public void syncTables( for (TableIdentifier tableId : tableSyncPlan.entitiesToCreate()) { try { - Table table = sourceIcebergCatalog.loadTable(tableId); + Table table = sourceIcebergCatalogService.loadTable(tableId); if (table instanceof BaseTable baseTable) { - targetIcebergCatalog.registerTable( + targetIcebergCatalogService.registerTable( tableId, baseTable.operations().current().metadataFileLocation()); } else { throw new IllegalStateException("Cannot register table that does not extend BaseTable."); @@ -1238,16 +1165,16 @@ public void syncTables( try { Table table; - if (sourceIcebergCatalog instanceof PolarisCatalog polarisCatalog) { + if (sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) { String etag = etagManager.getETag(catalogName, tableId); - table = polarisCatalog.loadTable(tableId, etag); + table = polarisCatalogService.loadTable(tableId, etag); } else { - table = sourceIcebergCatalog.loadTable(tableId); + table = sourceIcebergCatalogService.loadTable(tableId); } if (table instanceof BaseTable baseTable) { - targetIcebergCatalog.dropTable(tableId, /* purge */ false); - targetIcebergCatalog.registerTable( + targetIcebergCatalogService.dropTableWithoutPurge(tableId); + targetIcebergCatalogService.registerTable( tableId, baseTable.operations().current().metadataFileLocation()); } else { throw new IllegalStateException("Cannot register table that does not extend BaseTable."); @@ -1286,7 +1213,7 @@ public void syncTables( for (TableIdentifier table : tableSyncPlan.entitiesToRemove()) { try { - targetIcebergCatalog.dropTable(table, /* purge */ false); + targetIcebergCatalogService.dropTableWithoutPurge(table); clientLogger.info( "Dropped table {} in namespace {} in catalog {}. - {}/{}", table, diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java index 28e83f31..8e9896a2 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java @@ -37,7 +37,7 @@ import org.apache.polaris.core.admin.model.Principal; import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.tools.sync.polaris.PolarisService; +import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService; /** * Service class to facilitate the access control needs of the synchronization. This involves @@ -46,9 +46,9 @@ */ public class AccessControlService { - private final PolarisService polaris; + private final PolarisApiService polaris; - public AccessControlService(PolarisService polaris) { + public AccessControlService(PolarisApiService polaris) { this.polaris = polaris; } @@ -73,7 +73,7 @@ public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) { && principal.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { if (replace) { // drop existing omnipotent principal in preparation for replacement - polaris.removePrincipal(principal.getName()); + polaris.dropPrincipal(principal.getName()); } else { // we cannot create another omnipotent principal and cannot replace the existing, fail throw new IllegalStateException( @@ -86,7 +86,7 @@ public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) { } // existing principal with identifying property does not exist, create a new one - return polaris.createPrincipal(omnipotentPrincipalPrototype, false); + return polaris.createPrincipal(omnipotentPrincipalPrototype); } /** @@ -97,7 +97,7 @@ public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) { */ public PrincipalRole getOmnipotentPrincipalRoleForPrincipal(String principalName) { List principalRolesAssigned = - polaris.listPrincipalRolesAssignedForPrincipal(principalName); + polaris.listPrincipalRolesAssigned(principalName); return principalRolesAssigned.stream() .filter( @@ -137,7 +137,7 @@ public PrincipalRole createAndAssignPrincipalRole( && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { // replace existing principal role if exists if (replace) { - polaris.removePrincipalRole(principalRole.getName()); + polaris.dropPrincipalRole(principalRole.getName()); } else { throw new IllegalStateException( "Not permitted to replace existing omnipotent principal role, but omnipotent " @@ -148,9 +148,8 @@ public PrincipalRole createAndAssignPrincipalRole( } } - polaris.createPrincipalRole(omnipotentPrincipalRole, false); - polaris.assignPrincipalRoleToPrincipal( - omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName()); + polaris.createPrincipalRole(omnipotentPrincipalRole); + polaris.assignPrincipalRole(omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName()); return omnipotentPrincipalRole; } @@ -172,7 +171,7 @@ public CatalogRole createAndAssignCatalogRole( if (catalogRole.getProperties() != null && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) { if (replace) { - polaris.removeCatalogRole(catalogName, catalogRole.getName()); + polaris.dropCatalogRole(catalogName, catalogRole.getName()); } else { throw new IllegalStateException( "Not permitted to replace existing omnipotent catalog role for catalog " @@ -189,9 +188,8 @@ public CatalogRole createAndAssignCatalogRole( .name(omnipotentPrincipalRole.getName()) .putPropertiesItem(OMNIPOTENCE_PROPERTY, ""); - polaris.createCatalogRole(catalogName, omnipotentCatalogRole, false /* overwrite */); - polaris.assignCatalogRole( - omnipotentPrincipalRole.getName(), catalogName, omnipotentCatalogRole.getName()); + polaris.createCatalogRole(catalogName, omnipotentCatalogRole); + polaris.assignCatalogRole(omnipotentPrincipalRole.getName(), catalogName, omnipotentCatalogRole.getName()); return omnipotentCatalogRole; } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java new file mode 100644 index 00000000..a597d053 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java @@ -0,0 +1,35 @@ +package org.apache.polaris.tools.sync.polaris.service; + +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; + +import java.util.List; +import java.util.Map; + +/** + * Wrapper around {@link org.apache.iceberg.catalog.Catalog} that exposes functionality + * that uses multiple Iceberg operations. For example, cascading drops of namespaces. + */ +public interface IcebergCatalogService { + + // NAMESPACES + List listNamespaces(Namespace parentNamespace); + Map loadNamespaceMetadata(Namespace namespace); + void createNamespace(Namespace namespace, Map namespaceMetadata); + void setNamespaceProperties(Namespace namespace, Map namespaceProperties); + + /** + * Drop a namespace by first dropping all nested namespaces and tables underneath the namespace + * hierarchy. The empty namespace will not be dropped. + * @param namespace the namespace to drop + */ + void dropNamespaceCascade(Namespace namespace); + + // TABLES + List listTables(Namespace namespace); + Table loadTable(TableIdentifier tableIdentifier); + void registerTable(TableIdentifier tableIdentifier, String metadataFileLocation); + void dropTableWithoutPurge(TableIdentifier tableIdentifier); + +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java new file mode 100644 index 00000000..59dddfb8 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java @@ -0,0 +1,67 @@ +package org.apache.polaris.tools.sync.polaris.service; + +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +import java.util.List; +import java.util.Map; + +/** + * Generic wrapper for a Polaris entity store. + */ +public interface PolarisService { + + /** + * Called to perform initializing tasks for a Polaris entity store. + * @param properties the properties used to initialize the service + * @throws Exception + */ + void initialize(Map properties) throws Exception; + + // PRINCIPALS + List listPrincipals(); + Principal getPrincipal(String principalName); + PrincipalWithCredentials createPrincipal(Principal principal); + void dropPrincipal(String principalName); + + // PRINCIPAL ROLES + List listPrincipalRoles(); + PrincipalRole getPrincipalRole(String principalRoleName); + void createPrincipalRole(PrincipalRole principalRole); + void dropPrincipalRole(String principalRoleName); + + // ASSIGNMENT OF PRINCIPAL ROLES TO PRINCIPALS + List listPrincipalRolesAssigned(String principalName); + void assignPrincipalRole(String principalName, String principalRoleName); + void revokePrincipalRole(String principalName, String principalRoleName); + + // CATALOGS + List listCatalogs(); + Catalog getCatalog(String catalogName); + void createCatalog(Catalog catalog); + void dropCatalogCascade(String catalogName); + + // CATALOG ROLES + List listCatalogRoles(String catalogName); + CatalogRole getCatalogRole(String catalogName, String catalogRoleName); + void createCatalogRole(String catalogName, CatalogRole catalogRole); + void dropCatalogRole(String catalogName, String catalogRoleName); + + // ASSIGNMENT OF CATALOG ROLES TO CATALOGS + List listAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName); + void assignCatalogRole(String principalRoleName, String catalogName, String catalogRoleName); + void revokeCatalogRole(String principalRoleName, String catalogName, String catalogRoleName); + + // GRANTS + List listGrants(String catalogName, String catalogRoleName); + void addGrant(String catalogName, String catalogRoleName, GrantResource grant); + void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant); + + // ICEBERG + IcebergCatalogService initializeIcebergCatalogService(String catalogName); + +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java new file mode 100644 index 00000000..e1f25d40 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java @@ -0,0 +1,256 @@ +package org.apache.polaris.tools.sync.polaris.service.impl; + +import org.apache.http.HttpHeaders; +import org.apache.iceberg.catalog.Namespace; +import org.apache.polaris.core.admin.model.AddGrantRequest; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; +import org.apache.polaris.core.admin.model.CreatePrincipalRequest; +import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; +import org.apache.polaris.core.admin.model.RevokeGrantRequest; +import org.apache.polaris.management.ApiClient; +import org.apache.polaris.management.client.PolarisManagementDefaultApi; +import org.apache.polaris.tools.sync.polaris.access.AccessControlService; +import org.apache.polaris.tools.sync.polaris.http.OAuth2Util; +import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService; +import org.apache.polaris.tools.sync.polaris.service.PolarisService; + +import java.util.List; +import java.util.Map; + +public class PolarisApiService implements PolarisService { + + private String baseUrl = null; + + private PolarisManagementDefaultApi api = null; + + private AccessControlService accessControlService = null; + + private PrincipalWithCredentials omnipotentPrincipal = null; + + private PrincipalRole omnipotentPrincipalRole = null; + + private boolean icebergWriteAccess = false; + + public PolarisApiService() {} + + @Override + public void initialize(Map properties) throws Exception { + String baseUrl = properties.get("base-url"); + String token = properties.get("bearer-token"); + + if (token == null) { + String oauth2ServerUri = properties.get("oauth2-server-uri"); + String clientId = properties.get("client-id"); + String clientSecret = properties.get("client-secret"); + String scope = properties.get("scope"); + + token = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope); + } + + String bearerToken = token; // to make it effectively final to use it in a lambda + + ApiClient client = new ApiClient(); + client.updateBaseUri(baseUrl + "/api/management/v1"); + + // TODO: Add token refresh + client.setRequestInterceptor(requestBuilder -> + requestBuilder.header(HttpHeaders.AUTHORIZATION, "Bearer " + bearerToken)); + + this.baseUrl = baseUrl; + this.api = new PolarisManagementDefaultApi(client); + + this.omnipotentPrincipal = new PrincipalWithCredentials() + .principal(new Principal().name(properties.get("omnipotent-principal-name"))) + .credentials(new PrincipalWithCredentialsCredentials() + .clientId(properties.get("omnipotent-principal-client-id")) + .clientSecret(properties.get("omnipotent-principal-client-secret"))); + + this.accessControlService = new AccessControlService(this); + this.icebergWriteAccess = Boolean.parseBoolean(properties.get("iceberg-write-access")); + } + + @Override + public List listPrincipals() { + return this.api.listPrincipals().getPrincipals(); + } + + @Override + public Principal getPrincipal(String principalName) { + return this.api.getPrincipal(principalName); + } + + @Override + public PrincipalWithCredentials createPrincipal(Principal principal) { + CreatePrincipalRequest request = new CreatePrincipalRequest().principal(principal); + return this.api.createPrincipal(request); + } + + @Override + public void dropPrincipal(String principalName) { + this.api.deletePrincipal(principalName); + } + + @Override + public List listPrincipalRoles() { + return this.api.listPrincipalRoles().getRoles(); + } + + @Override + public PrincipalRole getPrincipalRole(String principalRoleName) { + return this.api.getPrincipalRole(principalRoleName); + } + + @Override + public void createPrincipalRole(PrincipalRole principalRole) { + CreatePrincipalRoleRequest request = new CreatePrincipalRoleRequest().principalRole(principalRole); + this.api.createPrincipalRole(request); + } + + @Override + public void dropPrincipalRole(String principalRoleName) { + this.api.deletePrincipalRole(principalRoleName); + } + + @Override + public List listPrincipalRolesAssigned(String principalName) { + return this.api.listPrincipalRolesAssigned(principalName).getRoles(); + } + + @Override + public void assignPrincipalRole(String principalName, String principalRoleName) { + GrantPrincipalRoleRequest request = new GrantPrincipalRoleRequest() + .principalRole(new PrincipalRole().name(principalRoleName)); + this.api.assignPrincipalRole(principalName, request); + } + + @Override + public void revokePrincipalRole(String principalName, String principalRoleName) { + this.api.revokePrincipalRole(principalName, principalRoleName); + } + + @Override + public List listCatalogs() { + return this.api.listCatalogs().getCatalogs(); + } + + @Override + public Catalog getCatalog(String catalogName) { + return this.api.getCatalog(catalogName); + } + + @Override + public void createCatalog(Catalog catalog) { + CreateCatalogRequest request = new CreateCatalogRequest().catalog(catalog); + this.api.createCatalog(request); + setupOmnipotentCatalogRoleIfNotExists(catalog.getName()); + } + + @Override + public void dropCatalogCascade(String catalogName) { + setupOmnipotentCatalogRoleIfNotExists(catalogName); + IcebergCatalogService icebergCatalogService = this.initializeIcebergCatalogService(catalogName); + + // drop all namespaces within catalog + icebergCatalogService.dropNamespaceCascade(Namespace.empty()); + + List catalogRoles = this.listCatalogRoles(catalogName); + + // drop all catalog roles within catalog + for (CatalogRole catalogRole : catalogRoles) { + if (!catalogRole.getName().equals("catalog_admin")) { + this.dropCatalogRole(catalogName, catalogRole.getName()); + } + } + + this.api.deleteCatalog(catalogName); + } + + @Override + public List listCatalogRoles(String catalogName) { + return this.api.listCatalogRoles(catalogName).getRoles(); + } + + @Override + public CatalogRole getCatalogRole(String catalogName, String catalogRoleName) { + return this.api.getCatalogRole(catalogName, catalogRoleName); + } + + @Override + public void createCatalogRole(String catalogName, CatalogRole catalogRole) { + CreateCatalogRoleRequest request = new CreateCatalogRoleRequest().catalogRole(catalogRole); + this.api.createCatalogRole(catalogName, request); + } + + @Override + public void dropCatalogRole(String catalogName, String catalogRoleName) { + this.api.deleteCatalogRole(catalogName, catalogRoleName); + } + + @Override + public List listAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName) { + return this.api.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName).getRoles(); + } + + @Override + public void assignCatalogRole(String principalRoleName, String catalogName, String catalogRoleName) { + GrantCatalogRoleRequest request = new GrantCatalogRoleRequest() + .catalogRole(new CatalogRole().name(catalogRoleName)); + this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request); + } + + @Override + public void revokeCatalogRole(String principalRoleName, String catalogName, String catalogRoleName) { + this.api.revokeCatalogRoleFromPrincipalRole(principalRoleName, catalogName, catalogRoleName); + } + + @Override + public List listGrants(String catalogName, String catalogRoleName) { + return this.api.listGrantsForCatalogRole(catalogName, catalogRoleName).getGrants(); + } + + @Override + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + AddGrantRequest request = new AddGrantRequest().grant(grant); + this.api.addGrantToCatalogRole(catalogName, catalogRoleName, request); + } + + @Override + public void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant) { + RevokeGrantRequest request = new RevokeGrantRequest().grant(grant); + this.api.revokeGrantFromCatalogRole(catalogName, catalogRoleName, false, request); + } + + private void setupOmnipotentCatalogRoleIfNotExists(String catalogName) { + if (this.omnipotentPrincipalRole == null) { + this.omnipotentPrincipalRole = + this.accessControlService.getOmnipotentPrincipalRoleForPrincipal( + omnipotentPrincipal.getPrincipal().getName()); + } + + if (!this.accessControlService.omnipotentCatalogRoleExists(catalogName)) { + this.accessControlService.setupOmnipotentRoleForCatalog( + catalogName, + omnipotentPrincipalRole, + false /* replace */, + icebergWriteAccess /* withWriteAccess */ + ); + } + } + + @Override + public IcebergCatalogService initializeIcebergCatalogService(String catalogName) { + setupOmnipotentCatalogRoleIfNotExists(catalogName); + return new PolarisIcebergCatalogService(baseUrl + "/api/catalog", catalogName, omnipotentPrincipal); + } + +} diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java new file mode 100644 index 00000000..e0c23975 --- /dev/null +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java @@ -0,0 +1,131 @@ +package org.apache.polaris.tools.sync.polaris.service.impl; + +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; +import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PolarisIcebergCatalogService implements IcebergCatalogService { + + private final PolarisCatalog catalog; + + public PolarisIcebergCatalogService(String uri, String catalogName, PrincipalWithCredentials migratorPrincipal) { + Map catalogProperties = new HashMap<>(); + catalogProperties.put("uri", uri); + catalogProperties.put("warehouse", catalogName); + + String clientId = migratorPrincipal.getCredentials().getClientId(); + String clientSecret = migratorPrincipal.getCredentials().getClientSecret(); + catalogProperties.putIfAbsent( + "credential", String.format("%s:%s", clientId, clientSecret)); + catalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL"); + + this.catalog = (PolarisCatalog) CatalogUtil.loadCatalog( + PolarisCatalog.class.getName(), + "SOURCE_CATALOG_REST_" + catalogName, + catalogProperties, + null + ); + } + + @Override + public List listNamespaces(Namespace parentNamespace) { + return this.catalog.listNamespaces(parentNamespace); + } + + /** + * List all namespaces in hierarchy underneath a particular namespace in addition to all + * immediate children. + * @param parentNamespace the namespace to search for child namespaces under + * @return all child namespaces in hierarchy + */ + private List listAllChildNamespaces(Namespace parentNamespace) { + List immediateChildren = this.listNamespaces(parentNamespace); + + List allChildNamespaces = new ArrayList<>(immediateChildren); + + for (Namespace childNamespace : immediateChildren) { + allChildNamespaces.addAll(this.listAllChildNamespaces(childNamespace)); + } + + return allChildNamespaces; + } + + @Override + public Map loadNamespaceMetadata(Namespace namespace) { + return this.catalog.loadNamespaceMetadata(namespace); + } + + @Override + public void createNamespace(Namespace namespace, Map namespaceMetadata) { + this.catalog.createNamespace(namespace, namespaceMetadata); + } + + @Override + public void setNamespaceProperties(Namespace namespace, Map namespaceProperties) { + this.catalog.setProperties(namespace, namespaceProperties); + } + + @Override + public void dropNamespaceCascade(Namespace namespace) { + List allChildNamespaces = this.listAllChildNamespaces(namespace); + + List tables = new ArrayList<>(); + + for (Namespace childNamespace : allChildNamespaces) { + tables.addAll(this.catalog.listTables(childNamespace)); + } + + if (!namespace.isEmpty()) { + tables.addAll(this.catalog.listTables(namespace)); + } + + for (TableIdentifier tableIdentifier : tables) { + this.catalog.dropTable(tableIdentifier); + } + + // go over in reverse order of namespaces since we discover namespaces + // in the parent -> child order, so we need to drop all children + // before we can drop the parent + for (Namespace childNamespace : allChildNamespaces.reversed()) { + this.catalog.dropNamespace(childNamespace); + } + + if (!namespace.isEmpty()) { + this.catalog.dropNamespace(namespace); + } + } + + @Override + public List listTables(Namespace namespace) { + return this.catalog.listTables(namespace); + } + + @Override + public Table loadTable(TableIdentifier tableIdentifier) { + return this.catalog.loadTable(tableIdentifier); + } + + public Table loadTable(TableIdentifier tableIdentifier, String etag) { + return this.catalog.loadTable(tableIdentifier, etag); + } + + @Override + public void registerTable(TableIdentifier tableIdentifier, String metadataFileLocation) { + this.catalog.registerTable(tableIdentifier, metadataFileLocation); + } + + @Override + public void dropTableWithoutPurge(TableIdentifier tableIdentifier) { + this.catalog.dropTable(tableIdentifier, false /* purge */); + } + +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java index a1b0a303..df17a719 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; @@ -31,7 +32,8 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.access.AccessControlService; -import org.apache.polaris.tools.sync.polaris.options.PolarisOptions; +import org.apache.polaris.tools.sync.polaris.service.PolarisService; +import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -51,8 +53,11 @@ public class CreateOmnipotentPrincipalCommand implements Callable { private final Logger consoleLog = LoggerFactory.getLogger("console-log"); - @CommandLine.ArgGroup(exclusive = false, multiplicity = "1", heading = "Polaris options: %n") - private PolarisOptions options; + @CommandLine.Option( + names = {"--polaris-api-connection-properties"}, + description = "The connection properties to connect to the Polaris API." + ) + private Map polarisApiConnectionProperties; @CommandLine.Option( names = {"--replace"}, @@ -79,8 +84,17 @@ public class CreateOmnipotentPrincipalCommand implements Callable { @Override public Integer call() throws Exception { - PolarisService polaris = options.buildService(); - AccessControlService accessControlService = new AccessControlService(polaris); + polarisApiConnectionProperties.putIfAbsent("iceberg-write-access", String.valueOf(withWriteAccess)); + + PolarisService polaris = PolarisServiceFactory.createPolarisService( + PolarisServiceFactory.ServiceType.API, + withWriteAccess + ? PolarisServiceFactory.EndpointType.TARGET + : PolarisServiceFactory.EndpointType.SOURCE, + polarisApiConnectionProperties + ); + + AccessControlService accessControlService = new AccessControlService((PolarisApiService) polaris); PrincipalWithCredentials principalWithCredentials; @@ -136,9 +150,10 @@ public Integer call() throws Exception { "Failed to setup omnipotent catalog role for catalog {} with {} access. - {}/{}", catalog.getName(), permissionLevel, - completedCatalogSetups.getAndIncrement(), + completedCatalogSetups.incrementAndGet(), catalogs.size(), e); + return; } consoleLog.info( diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java new file mode 100644 index 00000000..2107b0d4 --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java @@ -0,0 +1,40 @@ +package org.apache.polaris.tools.sync.polaris; + +import org.apache.polaris.tools.sync.polaris.service.PolarisService; +import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService; + +import java.util.Map; + +public class PolarisServiceFactory { + + public enum ServiceType { + API + } + + public enum EndpointType { + SOURCE, + TARGET + } + + public static PolarisService createPolarisService( + ServiceType serviceType, + EndpointType endpointType, + Map properties + ) { + PolarisService service = switch (serviceType) { + case API -> { + properties.putIfAbsent("iceberg-write-access", String.valueOf(endpointType == EndpointType.TARGET)); + yield new PolarisApiService(); + } + }; + + try { + service.initialize(properties); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return service; + } + +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 768c9ee9..cccd82be 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -21,18 +21,15 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.util.Map; import java.util.concurrent.Callable; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager; -import org.apache.polaris.tools.sync.polaris.options.SourceOmniPotentPrincipalOptions; -import org.apache.polaris.tools.sync.polaris.options.SourcePolarisOptions; -import org.apache.polaris.tools.sync.polaris.options.TargetOmnipotentPrincipal; -import org.apache.polaris.tools.sync.polaris.options.TargetPolarisOptions; import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; +import org.apache.polaris.tools.sync.polaris.service.PolarisService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -50,29 +47,33 @@ public class SyncPolarisCommand implements Callable { private final Logger consoleLog = LoggerFactory.getLogger("console-log"); - @CommandLine.ArgGroup( - exclusive = false, - multiplicity = "1", - heading = "Source Polaris options: %n") - private SourcePolarisOptions sourcePolarisOptions; - - @CommandLine.ArgGroup( - exclusive = false, - multiplicity = "1", - heading = "Target Polaris options: %n") - private TargetPolarisOptions targetPolarisOptions; - - @CommandLine.ArgGroup( - exclusive = false, - multiplicity = "1", - heading = "Source Polaris Omnipotent Principal Options: %n") - private SourceOmniPotentPrincipalOptions sourceOmniPotentPrincipalOptions; - - @CommandLine.ArgGroup( - exclusive = false, - multiplicity = "1", - heading = "Target Polaris Omnipotent Principal Options: %n") - private TargetOmnipotentPrincipal targetOmniPotentPrincipalOptions; + @CommandLine.Option( + names = {"--source-type"}, + required = true, + description = "The type of the Polaris entity source. One of { API }" + ) + private PolarisServiceFactory.ServiceType sourceType; + + @CommandLine.Option( + names = {"--source-properties"}, + required = true, + description = "The properties to initialize the Polaris entity source." + ) + private Map sourceProperties; + + @CommandLine.Option( + names = {"--target-type"}, + required = true, + description = "The type of the Polaris entity target. One of { API }" + ) + private PolarisServiceFactory.ServiceType targetType; + + @CommandLine.Option( + names = {"--target-properties"}, + required = true, + description = "The properties to initialize the Polaris entity target." + ) + private Map targetProperties; @CommandLine.Option( names = {"--etag-file"}, @@ -91,18 +92,14 @@ public class SyncPolarisCommand implements Callable { @Override public Integer call() throws Exception { SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner(); - SynchronizationPlanner modificationAwareSourceParityPlanner = - new ModificationAwarePlanner(sourceParityPlanner); - SynchronizationPlanner accessControlAwarePlanner = - new AccessControlAwarePlanner(modificationAwareSourceParityPlanner); + SynchronizationPlanner modificationAwareSourceParityPlanner = new ModificationAwarePlanner(sourceParityPlanner); + SynchronizationPlanner accessControlAwarePlanner = new AccessControlAwarePlanner(modificationAwareSourceParityPlanner); - PolarisService source = sourcePolarisOptions.buildService(); - PolarisService target = targetPolarisOptions.buildService(); - PrincipalWithCredentials sourceOmnipotentPrincipal = - sourceOmniPotentPrincipalOptions.buildPrincipalWithCredentials(); - PrincipalWithCredentials targetOmniPotentPrincipal = - targetOmniPotentPrincipalOptions.buildPrincipalWithCredentials(); + PolarisService source = PolarisServiceFactory.createPolarisService( + sourceType, PolarisServiceFactory.EndpointType.SOURCE, sourceProperties); + PolarisService target = PolarisServiceFactory.createPolarisService( + sourceType, PolarisServiceFactory.EndpointType.TARGET, targetProperties); ETagManager etagService; @@ -130,8 +127,6 @@ public Integer call() throws Exception { new PolarisSynchronizer( consoleLog, accessControlAwarePlanner, - sourceOmnipotentPrincipal, - targetOmniPotentPrincipal, source, target, etagService); diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java deleted file mode 100644 index 87766d57..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BaseOmnipotentPrincipalOptions.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; - -/** - * Base options class to define the common set of omnipotent principal authentication and connection properties. - * Can be used to give options different names based on the command while still ensuring they all - * satisfy the same sets of necessary properties. - */ -public abstract class BaseOmnipotentPrincipalOptions { - - protected static final String PRINCIPAL_NAME = "omni-principal-name"; - - protected static final String CLIENT_ID = "omni-client-id"; - - protected static final String CLIENT_SECRET = "omni-client-secret"; - - protected String principalName; - - protected String clientId; - - protected String clientSecret; - - public abstract void setPrincipalName(String principalName); - - public abstract void setClientId(String clientId); - - public abstract void setClientSecret(String clientSecret); - - public PrincipalWithCredentials buildPrincipalWithCredentials() { - return new PrincipalWithCredentials() - .principal(new Principal().name(principalName)) - .credentials( - new PrincipalWithCredentialsCredentials() - .clientId(clientId) - .clientSecret(clientSecret)); - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java deleted file mode 100644 index 19183b36..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/BasePolarisOptions.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import java.io.IOException; -import org.apache.polaris.tools.sync.polaris.PolarisService; -import org.apache.polaris.tools.sync.polaris.PolarisServiceFactory; - -/** - * Base options class to define the common set of Polaris service admin authentication and connection properties. - * Can be used to give options different names based on the command while still ensuring they all - * satisfy the same sets of necessary properties. - */ -public abstract class BasePolarisOptions { - - protected static final String BASE_URL = "base-url"; - - protected static final String CLIENT_ID = "client-id"; - - protected static final String CLIENT_SECRET = "client-secret"; - - protected static final String SCOPE = "scope"; - - protected static final String OAUTH2_SERVER_URI = "oauth2-server-uri"; - - protected static final String ACCESS_TOKEN = "access-token"; - - protected String baseUrl; - - protected String oauth2ServerUri; - - protected String clientId; - - protected String clientSecret; - - protected String scope; - - protected String accessToken; - - public abstract String getServiceName(); - - public abstract void setBaseUrl(String baseUrl); - - public abstract void setOauth2ServerUri(String oauth2ServerUri); - - public abstract void setClientId(String clientId); - - public abstract void setClientSecret(String clientSecret); - - public abstract void setScope(String scope); - - public abstract void setAccessToken(String accessToken); - - public PolarisService buildService() throws IOException { - if (accessToken != null) { - return PolarisServiceFactory.newPolarisService(baseUrl, accessToken); - } - return PolarisServiceFactory.newPolarisService( - baseUrl, oauth2ServerUri, clientId, clientSecret, scope); - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java deleted file mode 100644 index 571aac6a..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/PolarisOptions.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import picocli.CommandLine; - -/** - * Options for a generic Polaris instance. Use this for commands that only have to - * access one Polaris instance. - */ -public class PolarisOptions extends BasePolarisOptions { - - @Override - public String getServiceName() { - return "polaris"; - } - - @CommandLine.Option( - names = "--polaris-" + BASE_URL, - required = true, - description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") - @Override - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } - - @CommandLine.Option( - names = "--polaris-" + OAUTH2_SERVER_URI, - description = { - "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " - + "obtain an access token for the Polaris instance." - }) - @Override - public void setOauth2ServerUri(String oauth2ServerUri) { - this.oauth2ServerUri = oauth2ServerUri; - } - - @CommandLine.Option( - names = "--polaris-" + CLIENT_ID, - description = { - "(Note: required if access-token not provided) The client id for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @CommandLine.Option( - names = "--polaris-" + CLIENT_SECRET, - description = { - "(Note: required if access-token not provided) The client secret for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - } - - @CommandLine.Option( - names = "--polaris-" + SCOPE, - description = { - "(Note: required if access-token not provided) The scope that the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setScope(String scope) { - this.scope = scope; - } - - @CommandLine.Option( - names = "--polaris-" + ACCESS_TOKEN, - description = "The access token to authenticate to the Polaris instance") - @Override - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java deleted file mode 100644 index 28c1ee24..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourceOmniPotentPrincipalOptions.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import picocli.CommandLine; - -/** - * Prefixes omnipotent principal option names with "source" tags to identify that these are - * the connection properties for the source instance. - */ -public class SourceOmniPotentPrincipalOptions extends BaseOmnipotentPrincipalOptions { - - @CommandLine.Option( - names = "--source-" + PRINCIPAL_NAME, - required = true, - description = "The principal name of the source omnipotent principal.") - @Override - public void setPrincipalName(String principalName) { - this.principalName = principalName; - } - - @CommandLine.Option( - names = "--source-" + CLIENT_ID, - required = true, - description = "The client id of the source omnipotent principal.") - @Override - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @CommandLine.Option( - names = "--source-" + CLIENT_SECRET, - required = true, - description = "The client secret of the source omnipotent principal.") - @Override - public void setClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java deleted file mode 100644 index 85f8b2a9..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/SourcePolarisOptions.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import picocli.CommandLine; - -/** - * Prefixes service_admin connection option names with "source" tags to identify that these are - * the connection properties for the source instance. - */ -public class SourcePolarisOptions extends BasePolarisOptions { - - @Override - public String getServiceName() { - return "source"; - } - - @CommandLine.Option( - names = "--source-" + BASE_URL, - required = true, - description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") - @Override - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } - - @CommandLine.Option( - names = "--source-" + OAUTH2_SERVER_URI, - description = { - "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " - + "obtain an access token for the Polaris instance." - }) - @Override - public void setOauth2ServerUri(String oauth2ServerUri) { - this.oauth2ServerUri = oauth2ServerUri; - } - - @CommandLine.Option( - names = "--source-" + CLIENT_ID, - description = { - "(Note: required if access-token not provided) The client id for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @CommandLine.Option( - names = "--source-" + CLIENT_SECRET, - description = { - "(Note: required if access-token not provided) The client secret for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - } - - @CommandLine.Option( - names = "--source-" + SCOPE, - description = { - "(Note: required if access-token not provided) The scope that the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setScope(String scope) { - this.scope = scope; - } - - @CommandLine.Option( - names = "--source-" + ACCESS_TOKEN, - description = "The access token to authenticate to the Polaris instance") - @Override - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java deleted file mode 100644 index 7b5e9c01..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetOmnipotentPrincipal.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import picocli.CommandLine; - -/** - * Prefixes omnipotent principal option names with "target" tags to identify that these are - * the connection properties for the target instance. - */ -public class TargetOmnipotentPrincipal extends BaseOmnipotentPrincipalOptions { - - @CommandLine.Option( - names = "--target-" + PRINCIPAL_NAME, - required = true, - description = "The principal name of the source omnipotent principal.") - @Override - public void setPrincipalName(String principalName) { - this.principalName = principalName; - } - - @CommandLine.Option( - names = "--target-" + CLIENT_ID, - required = true, - description = "The client id of the source omnipotent principal.") - @Override - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @CommandLine.Option( - names = "--target-" + CLIENT_SECRET, - required = true, - description = "The client secret of the source omnipotent principal.") - @Override - public void setClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - } -} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java deleted file mode 100644 index 8e011758..00000000 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/options/TargetPolarisOptions.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.tools.sync.polaris.options; - -import picocli.CommandLine; - -/** - * Prefixes service_admin connection option names with "target" tags to identify that these are - * the connection properties for the target instance. - */ -public class TargetPolarisOptions extends BasePolarisOptions { - - @Override - public String getServiceName() { - return "target"; - } - - @CommandLine.Option( - names = "--target-" + BASE_URL, - required = true, - description = "The base url of the Polaris instance. Example: http://localhost:8181/polaris.") - @Override - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } - - @CommandLine.Option( - names = "--target-" + OAUTH2_SERVER_URI, - description = { - "(Note: required if access-token not provided) the oauth2-server-uri to authenticate against to " - + "obtain an access token for the Polaris instance." - }) - @Override - public void setOauth2ServerUri(String oauth2ServerUri) { - this.oauth2ServerUri = oauth2ServerUri; - } - - @CommandLine.Option( - names = "--target-" + CLIENT_ID, - description = { - "(Note: required if access-token not provided) The client id for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @CommandLine.Option( - names = "--target-" + CLIENT_SECRET, - description = { - "(Note: required if access-token not provided) The client secret for the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - } - - @CommandLine.Option( - names = "--target-" + SCOPE, - description = { - "(Note: required if access-token not provided) The scope that the principal the tool will assume" - + " to carry out the copy. This principal must have SERVICE_MANAGE_ACCESS level privileges." - }) - @Override - public void setScope(String scope) { - this.scope = scope; - } - - @CommandLine.Option( - names = "--target-" + ACCESS_TOKEN, - description = "The access token to authenticate to the Polaris instance") - @Override - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } -} From e7431f9394d09ec5ab87afdd8d8df3224ab7c52c Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Mon, 14 Apr 2025 19:22:50 -0700 Subject: [PATCH 08/18] Updated docs --- polaris-synchronizer/README.md | 64 ++++++++++--------- .../CreateOmnipotentPrincipalCommand.java | 11 +++- .../sync/polaris/SyncPolarisCommand.java | 20 +++++- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md index 8b8a2d9b..474bd568 100644 --- a/polaris-synchronizer/README.md +++ b/polaris-synchronizer/README.md @@ -56,13 +56,13 @@ and a catalog role per catalog with the appropriate grants to read all entities **Example:** Create a **read-only** principal on the source Polaris instance, and replace it if it already exists, with 10 concurrent catalog setup threads: ``` -java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \ ---polaris-client-id root \ ---polaris-client-secret \ ---polaris-base-url http://localhost:8181 \ ---polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ ---polaris-scope PRINCIPAL_ROLE:ALL \ ---replace \ # replace if it exists +java -jar cli/build/libs/polaris-synchronizer-cli.jar create-omnipotent-principal \ +--polaris-api-connection-properties base-url=http://localhost:8181 \ +--polaris-api-connection-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \ +--polaris-api-connection-properties client-id=root \ +--polaris-api-connection-properties client-secret= \ +--polaris-api-connection-properties scope=PRINCIPAL_ROLE:ALL \ +--replace \ # replace it if it already exists --concurrency 10 # 10 concurrent catalog setup threads ``` @@ -99,14 +99,15 @@ To create a read-write principal, we simply specify the `--write-access` option. **Example:** Create a read-write principal on your target Polaris instance, replacing it if it exists, with 10 concurrent catalog setup threads. ``` -java -jar polaris-synchronizer-cli.jar create-omnipotent-principal \ ---polaris-client-id root \ ---polaris-client-secret \ ---polaris-base-url http://localhost:8181 \ ---polaris-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ ---polaris-scope PRINCIPAL_ROLE:ALL \ ---replace \ # replace if it exists ---concurrency 10 \ # 10 concurrent catalog setup threads +java -jar cli/build/libs/polaris-synchronizer-cli.jar \ +create-omnipotent-principal \ +--polaris-api-connection-properties base-url=http://localhost:8181 \ +--polaris-api-connection-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \ +--polaris-api-connection-properties client-id=root \ +--polaris-api-connection-properties client-secret= \ +--polaris-api-connection-properties scope=PRINCIPAL_ROLE:ALL \ +--replace \ # replace if it already exists +--concurrency 10 \ # 10 concurrent catalog setup threads --write-access # give the principal write access to catalog internals ``` @@ -143,20 +144,25 @@ diff between the source and target Polaris instances. This can be achieved using **Example** Running the synchronization between source Polaris instance using an access token, and a target Polaris instance using client credentials. ``` -java -jar polaris-synchronizer-cli.jar sync-polaris \ ---source-base-url http://localhost:8182 \ ---source-access-token \ ---target-base-url http://localhost:8181 \ ---target-client-id root \ ---target-client-secret \ ---target-oauth2-server-uri http://localhost:8181/api/catalog/v1/oauth/tokens \ ---target-scope PRINCIPAL_ROLE:ALL \ ---source-omni-principal-name omnipotent-principal-XXXXX \ ---source-omni-client-id ff7s8f9asbX10 \ ---source-omni-client-secret \ ---target-omni-principal-name omnipotent-principal-YYYYY \ ---target-omni-client-id 0af20a3a0037a40d \ ---target-omni-client-secret +java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \ +--source-type API \ +--source-properties base-url=http://localhost:8181 \ +--source-properties client-id=root \ +--source-properties client-secret= \ +--source-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \ +--source-properties scope=PRINCIPAL_ROLE:ALL \ +--source-properties omnipotent-principal-name=omnipotent-principal-XXXXX \ +--source-properties omnipotent-principal-client-id=589550e8b23d271e \ +--source-properties omnipotent-principal-client-secret= \ +--target-type API \ +--target-properties base-url=http://localhost:5858 \ +--target-properties client-id=root \ +--target-properties client-secret= \ +--target-properties oauth2-server-uri=http://localhost:5858/api/catalog/v1/oauth/tokens \ +--target-properties scope=PRINCIPAL_ROLE:ALL \ +--target-properties omnipotent-principal-name=omnipotent-principal-YYYYY \ +--target-properties omnipotent-principal-client-id=9b8ac0f1e4e2e614 \ +--target-properties omnipotent-principal-client-secret= ``` > :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java index df17a719..b6625de0 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java @@ -55,7 +55,16 @@ public class CreateOmnipotentPrincipalCommand implements Callable { @CommandLine.Option( names = {"--polaris-api-connection-properties"}, - description = "The connection properties to connect to the Polaris API." + required = true, + description = "The connection properties to connect to the Polaris API." + + "\nProperties:" + + "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" + + "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " + + "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." + + "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" + + "\n\t- client-id: the client id belonging to a service admin to authenticate with" + + "\n\t- client-secret: the client secret belong to a service admin to authenticate with" + + "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" ) private Map polarisApiConnectionProperties; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index cccd82be..55805e6f 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -57,7 +57,15 @@ public class SyncPolarisCommand implements Callable { @CommandLine.Option( names = {"--source-properties"}, required = true, - description = "The properties to initialize the Polaris entity source." + description = "Properties to initialize Polaris entity source." + + "\nProperties (source-type=API):" + + "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" + + "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " + + "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." + + "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" + + "\n\t- client-id: the client id belonging to a service admin to authenticate with" + + "\n\t- client-secret: the client secret belong to a service admin to authenticate with" + + "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" ) private Map sourceProperties; @@ -71,7 +79,15 @@ public class SyncPolarisCommand implements Callable { @CommandLine.Option( names = {"--target-properties"}, required = true, - description = "The properties to initialize the Polaris entity target." + description = "Properties to initialize Polaris entity target." + + "\nProperties (target-type=API):" + + "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" + + "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " + + "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." + + "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" + + "\n\t- client-id: the client id belonging to a service admin to authenticate with" + + "\n\t- client-secret: the client secret belong to a service admin to authenticate with" + + "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" ) private Map targetProperties; From 8604742898d231f62bb3a76cf1ff30203c98e0c7 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Tue, 15 Apr 2025 10:50:33 -0700 Subject: [PATCH 09/18] Remove type in options --- .../sync/polaris/SyncPolarisCommand.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 55805e6f..c7ae2d9d 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -47,18 +47,11 @@ public class SyncPolarisCommand implements Callable { private final Logger consoleLog = LoggerFactory.getLogger("console-log"); - @CommandLine.Option( - names = {"--source-type"}, - required = true, - description = "The type of the Polaris entity source. One of { API }" - ) - private PolarisServiceFactory.ServiceType sourceType; - @CommandLine.Option( names = {"--source-properties"}, required = true, description = "Properties to initialize Polaris entity source." + - "\nProperties (source-type=API):" + + "\nProperties:" + "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" + "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " + "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." + @@ -69,18 +62,11 @@ public class SyncPolarisCommand implements Callable { ) private Map sourceProperties; - @CommandLine.Option( - names = {"--target-type"}, - required = true, - description = "The type of the Polaris entity target. One of { API }" - ) - private PolarisServiceFactory.ServiceType targetType; - @CommandLine.Option( names = {"--target-properties"}, required = true, description = "Properties to initialize Polaris entity target." + - "\nProperties (target-type=API):" + + "\nProperties:" + "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" + "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " + "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." + @@ -113,9 +99,9 @@ public Integer call() throws Exception { PolarisService source = PolarisServiceFactory.createPolarisService( - sourceType, PolarisServiceFactory.EndpointType.SOURCE, sourceProperties); + PolarisServiceFactory.ServiceType.API, PolarisServiceFactory.EndpointType.SOURCE, sourceProperties); PolarisService target = PolarisServiceFactory.createPolarisService( - sourceType, PolarisServiceFactory.EndpointType.TARGET, targetProperties); + PolarisServiceFactory.ServiceType.API, PolarisServiceFactory.EndpointType.TARGET, targetProperties); ETagManager etagService; From 67c3e4154dea04a3e9b82d04ec53707ddda64555 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Tue, 15 Apr 2025 10:54:36 -0700 Subject: [PATCH 10/18] update docs --- polaris-synchronizer/README.md | 2 -- .../tools/sync/polaris/SyncPolarisCommand.java | 12 ++++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md index 474bd568..c536d17d 100644 --- a/polaris-synchronizer/README.md +++ b/polaris-synchronizer/README.md @@ -145,7 +145,6 @@ diff between the source and target Polaris instances. This can be achieved using using client credentials. ``` java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \ ---source-type API \ --source-properties base-url=http://localhost:8181 \ --source-properties client-id=root \ --source-properties client-secret= \ @@ -154,7 +153,6 @@ java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \ --source-properties omnipotent-principal-name=omnipotent-principal-XXXXX \ --source-properties omnipotent-principal-client-id=589550e8b23d271e \ --source-properties omnipotent-principal-client-secret= \ ---target-type API \ --target-properties base-url=http://localhost:5858 \ --target-properties client-id=root \ --target-properties client-secret= \ diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index c7ae2d9d..07d8cb77 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -58,7 +58,11 @@ public class SyncPolarisCommand implements Callable { "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" + "\n\t- client-id: the client id belonging to a service admin to authenticate with" + "\n\t- client-secret: the client secret belong to a service admin to authenticate with" + - "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" + "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" + + "\nOmnipotent Principal Properties:" + + "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + + "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + + "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the source Polaris" ) private Map sourceProperties; @@ -73,7 +77,11 @@ public class SyncPolarisCommand implements Callable { "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" + "\n\t- client-id: the client id belonging to a service admin to authenticate with" + "\n\t- client-secret: the client secret belong to a service admin to authenticate with" + - "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" + "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" + + "\nOmnipotent Principal Properties:" + + "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + + "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + + "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the target Polaris" ) private Map targetProperties; From 2ea47edfa213981685cc9e17d493a65297237dd7 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Tue, 15 Apr 2025 13:03:34 -0700 Subject: [PATCH 11/18] Added license headers --- .../service/IcebergCatalogService.java | 19 +++++++++++++++++++ .../sync/polaris/service/PolarisService.java | 19 +++++++++++++++++++ .../service/impl/PolarisApiService.java | 19 +++++++++++++++++++ .../impl/PolarisIcebergCatalogService.java | 19 +++++++++++++++++++ .../sync/polaris/PolarisServiceFactory.java | 19 +++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java index a597d053..b08c640f 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.polaris.tools.sync.polaris.service; import org.apache.iceberg.Table; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java index 59dddfb8..f011880c 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.polaris.tools.sync.polaris.service; import org.apache.polaris.core.admin.model.Catalog; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java index e1f25d40..fb881579 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.polaris.tools.sync.polaris.service.impl; import org.apache.http.HttpHeaders; diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java index e0c23975..9b1c3cb7 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.polaris.tools.sync.polaris.service.impl; import org.apache.iceberg.CatalogUtil; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java index 2107b0d4..26e06fd1 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.polaris.tools.sync.polaris; import org.apache.polaris.tools.sync.polaris.service.PolarisService; From fbf51a5d7d8b41cfe6ed091f804b85d082f13166 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 09:48:43 -0700 Subject: [PATCH 12/18] Add hard failure flag --- .../sync/polaris/PolarisSynchronizer.java | 51 +++++++++++++++++++ .../sync/polaris/SyncPolarisCommand.java | 7 +++ 2 files changed, 58 insertions(+) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index 014ccf71..d2ece8dd 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -59,14 +59,18 @@ public class PolarisSynchronizer { private final ETagManager etagManager; + private boolean haltOnFailure; + public PolarisSynchronizer( Logger clientLogger, + boolean haltOnFailure, SynchronizationPlanner synchronizationPlanner, PolarisService source, PolarisService target, ETagManager etagManager) { this.clientLogger = clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger; + this.haltOnFailure = haltOnFailure; this.syncPlanner = synchronizationPlanner; this.source = source; this.target = target; @@ -93,6 +97,7 @@ public void syncPrincipals() { principalsSource = source.listPrincipals(); clientLogger.info("Listed {} principals from source.", principalsSource.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.info("Failed to list principals from source.", e); return; } @@ -103,6 +108,7 @@ public void syncPrincipals() { principalsTarget = target.listPrincipals(); clientLogger.info("Listed {} principals from target.", principalsTarget.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.info("Failed to list principals from target.", e); return; } @@ -138,6 +144,7 @@ public void syncPrincipals() { totalSyncsToComplete ); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to create principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); } @@ -155,6 +162,7 @@ public void syncPrincipals() { totalSyncsToComplete ); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to overwrite principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); } @@ -166,6 +174,7 @@ public void syncPrincipals() { clientLogger.info("Removed principal {} on target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to remove principal {} ont target. - {}/{}", principal.getName(), ++syncsCompleted, totalSyncsToComplete, e); } @@ -188,6 +197,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { clientLogger.info("Listed {} assigned principal-roles for principal {} from source.", assignedPrincipalRolesSource.size(), principalName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list assigned principal-roles for principal {} from source.", principalName, e); return; } @@ -199,6 +209,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { clientLogger.info("Listed {} assigned principal-roles for principal {} from target.", assignedPrincipalRolesTarget.size(), principalName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list assigned principal-roles for principal {} from target.", principalName, e); return; } @@ -231,6 +242,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } @@ -242,6 +254,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } @@ -253,6 +266,7 @@ public void syncAssignedPrincipalRolesForPrincipal(String principalName) { clientLogger.info("Revoked principal-role {} from principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to revoke principal-role {} to principal {}. - {}/{}", principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete); } @@ -267,6 +281,7 @@ public void syncPrincipalRoles() { principalRolesSource = source.listPrincipalRoles(); clientLogger.info("Listed {} principal-roles from source.", principalRolesSource.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list principal-roles from source.", e); return; } @@ -277,6 +292,7 @@ public void syncPrincipalRoles() { principalRolesTarget = target.listPrincipalRoles(); clientLogger.info("Listed {} principal-roles from target.", principalRolesTarget.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list principal-roles from target.", e); return; } @@ -310,6 +326,7 @@ public void syncPrincipalRoles() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to create principal-role {} on target. - {}/{}", principalRole.getName(), @@ -329,6 +346,7 @@ public void syncPrincipalRoles() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to overwrite principal-role {} on target. - {}/{}", principalRole.getName(), @@ -347,6 +365,7 @@ public void syncPrincipalRoles() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to remove principal-role {} on target. - {}/{}", principalRole.getName(), @@ -375,6 +394,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list assignee principal-roles for catalog-role {} in catalog {} from source.", catalogRoleName, @@ -394,6 +414,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list assignee principal-roles for catalog-role {} in catalog {} from target.", catalogRoleName, @@ -441,6 +462,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}", principalRole.getName(), @@ -464,6 +486,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}", principalRole.getName(), @@ -487,6 +510,7 @@ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to revoke principal-role {} from catalog-role {} in catalog {}. - {}/{}", principalRole.getName(), @@ -507,6 +531,7 @@ public void syncCatalogs() { catalogsSource = source.listCatalogs(); clientLogger.info("Listed {} catalogs from source.", catalogsSource.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list catalogs from source.", e); return; } @@ -517,6 +542,7 @@ public void syncCatalogs() { catalogsTarget = target.listCatalogs(); clientLogger.info("Listed {} catalogs from target.", catalogsTarget.size()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error("Failed to list catalogs from target.", e); return; } @@ -554,6 +580,7 @@ public void syncCatalogs() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to create catalog {}. - {}/{}", catalog.getName(), @@ -573,6 +600,7 @@ public void syncCatalogs() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to overwrite catalog {}. - {}/{}", catalog.getName(), @@ -591,6 +619,7 @@ public void syncCatalogs() { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to remove catalog {}. - {}/{}", catalog.getName(), @@ -611,6 +640,7 @@ public void syncCatalogs() { "Initialized Iceberg REST catalog for Polaris catalog {} on source.", catalog.getName()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to initialize Iceberg REST catalog for Polaris catalog {} on source.", catalog.getName(), @@ -626,6 +656,7 @@ public void syncCatalogs() { "Initialized Iceberg REST catalog for Polaris catalog {} on target.", catalog.getName()); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to initialize Iceberg REST catalog for Polaris catalog {} on target.", catalog.getName(), @@ -653,6 +684,7 @@ public void syncCatalogRoles(String catalogName) { catalogRolesSource.size(), catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list catalog-roles for catalog {} from source.", catalogName, e); return; @@ -667,6 +699,7 @@ public void syncCatalogRoles(String catalogName) { catalogRolesTarget.size(), catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list catalog-roles for catalog {} from target.", catalogName, e); return; @@ -713,6 +746,7 @@ public void syncCatalogRoles(String catalogName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to create catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -734,6 +768,7 @@ public void syncCatalogRoles(String catalogName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to overwrite catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -754,6 +789,7 @@ public void syncCatalogRoles(String catalogName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to remove catalog-role {} for catalog {}. - {}/{}", catalogRole.getName(), @@ -787,6 +823,7 @@ private void syncGrants(String catalogName, String catalogRoleName) { catalogRoleName, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list grants for catalog-role {} in catalog {} from source.", catalogRoleName, @@ -805,6 +842,7 @@ private void syncGrants(String catalogName, String catalogRoleName) { catalogRoleName, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list grants for catalog-role {} in catalog {} from target.", catalogRoleName, @@ -850,6 +888,7 @@ private void syncGrants(String catalogName, String catalogRoleName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}", grant.getType(), @@ -872,6 +911,7 @@ private void syncGrants(String catalogName, String catalogRoleName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}", grant.getType(), @@ -894,6 +934,7 @@ private void syncGrants(String catalogName, String catalogRoleName) { ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to revoke grant {} from catalog-role {} for catalog {}. - {}/{}", grant.getType(), @@ -929,6 +970,7 @@ public void syncNamespaces( parentNamespace, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list namespaces in namespace {} for catalog {} from source.", parentNamespace, @@ -947,6 +989,7 @@ public void syncNamespaces( parentNamespace, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list namespaces in namespace {} for catalog {} from target.", parentNamespace, @@ -984,6 +1027,7 @@ public void syncNamespaces( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to create namespace {} in namespace {} for catalog {} - {}/{}", namespace, @@ -1023,6 +1067,7 @@ public void syncNamespaces( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to overwrite namespace metadata {} in namespace {} for catalog {} - {}/{}", namespace, @@ -1045,6 +1090,7 @@ public void syncNamespaces( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}", namespace, @@ -1085,6 +1131,7 @@ public void syncTables( namespace, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list tables in namespace {} for catalog {} on source.", namespace, @@ -1103,6 +1150,7 @@ public void syncTables( namespace, catalogName); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to list tables in namespace {} for catalog {} on target.", namespace, @@ -1150,6 +1198,7 @@ public void syncTables( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to register table {} in namespace {} in catalog {}. - {}/{}", tableId, @@ -1200,6 +1249,7 @@ public void syncTables( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.error( "Failed to drop and re-register table {} in namespace {} in catalog {}. - {}/{}", tableId, @@ -1222,6 +1272,7 @@ public void syncTables( ++syncsCompleted, totalSyncsToComplete); } catch (Exception e) { + if (haltOnFailure) throw e; clientLogger.info( "Failed to drop table {} in namespace {} in catalog {}. - {}/{}", table, diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 07d8cb77..d27c2200 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -99,6 +99,12 @@ public class SyncPolarisCommand implements Callable { ) private boolean shouldSyncPrincipals; + @CommandLine.Option( + names = {"--halt-on-failure"}, + description = "Hard fail and stop the synchronization when an error occurs." + ) + private boolean haltOnFailure; + @Override public Integer call() throws Exception { SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner(); @@ -136,6 +142,7 @@ public Integer call() throws Exception { PolarisSynchronizer synchronizer = new PolarisSynchronizer( consoleLog, + haltOnFailure, accessControlAwarePlanner, source, target, From c4a253992163e71fc53dc4f1dfcba72b2533f71a Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 09:50:38 -0700 Subject: [PATCH 13/18] make flag final --- .../apache/polaris/tools/sync/polaris/PolarisSynchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java index d2ece8dd..20d35346 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java @@ -59,7 +59,7 @@ public class PolarisSynchronizer { private final ETagManager etagManager; - private boolean haltOnFailure; + private final boolean haltOnFailure; public PolarisSynchronizer( Logger clientLogger, From 5f1d35a9bb196000a20df979949700442c24c3ee Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 09:57:52 -0700 Subject: [PATCH 14/18] Added explanation to README.md --- polaris-synchronizer/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md index c536d17d..a38de28d 100644 --- a/polaris-synchronizer/README.md +++ b/polaris-synchronizer/README.md @@ -12,8 +12,10 @@ Polaris specific entities, like principal-roles, catalog-roles, grants. * **Migration:** A user may have an active Polaris deployment that they want to migrate to a managed cloud offering like [Snowflake Open Catalog](https://www.snowflake.com/en/product/features/open-catalog/). * **Preventing Vendor Lock-In:** A user may currently have a managed Polaris offering and want the freedom to switch providers or to host Polaris themselves. -* **Mirroring/Disaster Recovery:** Modern data solutions require employing redundancy to ensure no single point of - failure. The tool can be scheduled on a cron to run periodic incremental syncs. +* **Backup:** Modern data solutions often require employing redundancy. This tool can be run on a periodic cron to keep snapshots of a Polaris instance. + +In the case of migration to/from a cloud offering, access to the Polaris metastore is possibly limited or entirely restricted. +This tool instead uses the Polaris REST API to perform the migration/synchronization. The tool currently supports migrating the following Polaris Management entities: * Optionally, Principals (with `--sync-principals` flag). Credentials will be different on the target instance. From 4491f9f208a07b6a4bed25bc596371755af56c06 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 15:38:26 -0700 Subject: [PATCH 15/18] Make ETagManager configurable --- .../sync/polaris/catalog/ETagManager.java | 9 +++ .../sync/polaris/catalog/NoOpETagManager.java | 5 ++ .../tools/sync/polaris/CsvETagManager.java | 39 ++++++++--- .../sync/polaris/ETagManagerFactory.java | 67 +++++++++++++++++++ .../sync/polaris/SyncPolarisCommand.java | 29 ++++---- .../sync/polaris/ETagManagerFactoryTest.java | 41 ++++++++++++ 6 files changed, 167 insertions(+), 23 deletions(-) create mode 100644 polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java create mode 100644 polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java index ded0a866..832d75ce 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java @@ -20,12 +20,21 @@ import org.apache.iceberg.catalog.TableIdentifier; +import java.util.Map; + /** * Generic interface to provide and store ETags for tables within catalogs. This allows the storage * of the ETag to be completely independent from the tool. */ public interface ETagManager { + /** + * Used to initialize the instance for use. Should be called prior to + * calling any methods. + * @param properties properties to configure instance with + */ + void initialize(Map properties); + /** * Retrieves the ETag for the table. * diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java index ff8fa3d9..1bd1499e 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java @@ -20,9 +20,14 @@ import org.apache.iceberg.catalog.TableIdentifier; +import java.util.Map; + /** Implementation that returns nothing and stores no ETags. */ public class NoOpETagManager implements ETagManager { + @Override + public void initialize(Map properties) {} + @Override public String getETag(String catalogName, TableIdentifier tableIdentifier) { return null; diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java index bd477ac1..4bcaab5d 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java @@ -37,6 +37,8 @@ /** Implementation that stores/loads ETags to/from a CSV file. */ public class CsvETagManager implements ETagManager, Closeable { + public static final String CSV_FILE_PROPERTY = "csv-file"; + private static final String CATALOG_HEADER = "Catalog"; private static final String TABLE_ID_HEADER = "TableIdentifier"; @@ -45,32 +47,49 @@ public class CsvETagManager implements ETagManager, Closeable { private static final String[] HEADERS = {CATALOG_HEADER, TABLE_ID_HEADER, ETAG_HEADER}; - private final File file; + private File file; private final Map> tablesByCatalogName; - public CsvETagManager(File file) throws IOException { + public CsvETagManager() { this.tablesByCatalogName = new HashMap<>(); - this.file = file; + } + + @Override + public void initialize(Map properties) { + if (!properties.containsKey(CSV_FILE_PROPERTY)) { + throw new IllegalArgumentException("Missing required property " + CSV_FILE_PROPERTY); + } + + this.file = new File(properties.get(CSV_FILE_PROPERTY)); if (file.exists()) { CSVFormat readerCSVFormat = - CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get(); + CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get(); + + CSVParser parser; - CSVParser parser = - CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat); + try { + parser = CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat); + } catch (IOException e) { + throw new RuntimeException(e); + } - for (CSVRecord record : parser.getRecords()) { + for (CSVRecord record : parser.getRecords()) { this.tablesByCatalogName.putIfAbsent(record.get(CATALOG_HEADER), new HashMap<>()); TableIdentifier tableId = TableIdentifier.parse(record.get(TABLE_ID_HEADER)); this.tablesByCatalogName - .get(record.get(CATALOG_HEADER)) - .put(tableId, record.get(ETAG_HEADER)); + .get(record.get(CATALOG_HEADER)) + .put(tableId, record.get(ETAG_HEADER)); } - parser.close(); + try { + parser.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } } diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java new file mode 100644 index 00000000..9f1dc66a --- /dev/null +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java @@ -0,0 +1,67 @@ +package org.apache.polaris.tools.sync.polaris; + +import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; +import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager; + +import java.util.HashMap; +import java.util.Map; + +/** + * Factory class to construct configurable {@link ETagManager} implementations. + */ +public class ETagManagerFactory { + + /** + * Property that will hold class name for custom {@link ETagManager} implementation. + */ + public static final String CUSTOM_CLASS_NAME_PROPERTY = "custom-impl"; + + private ETagManagerFactory() {} + + /** + * Recognized types of {@link ETagManager} implementations + */ + public enum Type { + NONE, + FILE, + CUSTOM + } + + /** + * Construct a new {@link ETagManager} instance. + * @param type the recognized type of the {@link ETagManager} to construct + * @param properties properties to use when initializing the {@link ETagManager} + * @return the constructed and initialized {@link ETagManager} + */ + public static ETagManager createETagManager(Type type, Map properties) { + try { + properties = properties == null ? new HashMap<>() : properties; + + ETagManager manager = switch (type) { + case NONE -> new NoOpETagManager(); + case FILE -> new CsvETagManager(); + case CUSTOM -> { + String customManagerClassname = properties.get(CUSTOM_CLASS_NAME_PROPERTY); + + if (customManagerClassname == null) { + throw new IllegalArgumentException("Missing required property " + CUSTOM_CLASS_NAME_PROPERTY); + } + + Object custom = Class.forName(customManagerClassname).getDeclaredConstructor().newInstance(); + + if (custom instanceof ETagManager customManager) { + yield customManager; + } + + throw new InstantiationException("Custom ETagManager '" + customManagerClassname + "' does not implement ETagManager"); + } + }; + + manager.initialize(properties); + return manager; + } catch (Exception e) { + throw new RuntimeException("Failed to construct ETagManager", e); + } + } + +} diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index d27c2200..870a4b27 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -19,12 +19,10 @@ package org.apache.polaris.tools.sync.polaris; import java.io.Closeable; -import java.io.File; import java.io.IOException; import java.util.Map; import java.util.concurrent.Callable; import org.apache.polaris.tools.sync.polaris.catalog.ETagManager; -import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager; import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner; import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner; import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; @@ -86,9 +84,21 @@ public class SyncPolarisCommand implements Callable { private Map targetProperties; @CommandLine.Option( - names = {"--etag-file"}, - description = "The file path of the file to retrieve and store table ETags from.") - private String etagFilePath; + names = {"--etag-storage-type"}, + defaultValue = "NONE", + description = "One of { NONE, FILE, CUSTOM }. Default: NONE. The storage manager to use for storing ETags." + ) + private ETagManagerFactory.Type etagManagerType; + + @CommandLine.Option( + names = {"--etag-storage-properties"}, + description = "Properties to initialize ETag storage." + + "\nFor type FILE:" + + "\n\t- " + CsvETagManager.CSV_FILE_PROPERTY + ": The CSV file to read ETags from and write ETags to." + + "\nFor type CUSTOM:" + + "\n\t- " + ETagManagerFactory.CUSTOM_CLASS_NAME_PROPERTY+ ": The classname for the custom ETagManager implementation." + ) + private Map etagManagerProperties; @CommandLine.Option( names = {"--sync-principals"}, @@ -117,14 +127,7 @@ public Integer call() throws Exception { PolarisService target = PolarisServiceFactory.createPolarisService( PolarisServiceFactory.ServiceType.API, PolarisServiceFactory.EndpointType.TARGET, targetProperties); - ETagManager etagService; - - if (etagFilePath != null) { - File etagFile = new File(etagFilePath); - etagService = new CsvETagManager(etagFile); - } else { - etagService = new NoOpETagManager(); - } + ETagManager etagService = ETagManagerFactory.createETagManager(etagManagerType, etagManagerProperties); Runtime.getRuntime() .addShutdownHook( diff --git a/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java b/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java new file mode 100644 index 00000000..741d8133 --- /dev/null +++ b/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java @@ -0,0 +1,41 @@ +package org.apache.polaris.tools.sync.polaris; + +import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class ETagManagerFactoryTest { + + @Test + public void constructCsvManagerSuccessfully() { + // Pass correct properties for default file etag manager + ETagManagerFactory.createETagManager( + ETagManagerFactory.Type.FILE, Map.of(CsvETagManager.CSV_FILE_PROPERTY, "test.csv")); + } + + @Test + public void failToConstructCsvManager() { + // omit properties for default file etag manager + Assertions.assertThrows(Exception.class, () -> + ETagManagerFactory.createETagManager(ETagManagerFactory.Type.FILE, Map.of())); + } + + @Test + public void constructCustomETagManagerSuccessfully() { + // correctly construct custom manager by passing classname + ETagManagerFactory.createETagManager( + ETagManagerFactory.Type.CUSTOM, + Map.of(ETagManagerFactory.CUSTOM_CLASS_NAME_PROPERTY, NoOpETagManager.class.getName()) // use a recognized one for now + ); + } + + @Test + public void failToConstructCustomETagManager() { + // fail to construct custom etag manager by omitting custom classname from config + Assertions.assertThrows(Exception.class, () -> + ETagManagerFactory.createETagManager(ETagManagerFactory.Type.CUSTOM, Map.of())); + } + +} From 0525d9acfb72f4d0b24dd45442be496265fc4abd Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 17:21:25 -0700 Subject: [PATCH 16/18] Add configurable oauth server for omnipotent principal --- polaris-synchronizer/README.md | 4 +++- .../sync/polaris/catalog/PolarisCatalog.java | 3 ++- .../service/IcebergCatalogService.java | 2 +- .../service/impl/PolarisApiService.java | 7 ++++++- .../impl/PolarisIcebergCatalogService.java | 19 ++++++++++++++++--- .../sync/polaris/SyncPolarisCommand.java | 8 ++++++-- 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md index a38de28d..fb9e4f65 100644 --- a/polaris-synchronizer/README.md +++ b/polaris-synchronizer/README.md @@ -155,6 +155,7 @@ java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \ --source-properties omnipotent-principal-name=omnipotent-principal-XXXXX \ --source-properties omnipotent-principal-client-id=589550e8b23d271e \ --source-properties omnipotent-principal-client-secret= \ +--source-properties omnipotent-principal-oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \ --target-properties base-url=http://localhost:5858 \ --target-properties client-id=root \ --target-properties client-secret= \ @@ -162,7 +163,8 @@ java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \ --target-properties scope=PRINCIPAL_ROLE:ALL \ --target-properties omnipotent-principal-name=omnipotent-principal-YYYYY \ --target-properties omnipotent-principal-client-id=9b8ac0f1e4e2e614 \ ---target-properties omnipotent-principal-client-secret= +--target-properties omnipotent-principal-client-secret= \ +--target-properties omnipotent-principal-oauth2-server-uri=http://localhost:5858/api/catalog/v1/oauth/tokens ``` > :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java index 37b64e44..5c489aed 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java @@ -80,6 +80,8 @@ public void initialize(String name, Map props) { resourcePaths = ResourcePaths.forCatalogProperties(this.properties); } + super.initialize(name, props); + if (accessToken == null || httpClient == null || this.objectMapper == null) { String oauth2ServerUri = props.get("uri") + "/v1/oauth/tokens"; String credential = props.get("credential"); @@ -99,7 +101,6 @@ public void initialize(String name, Map props) { this.httpClient = HttpClient.newBuilder().build(); this.objectMapper = new ObjectMapper(); } - super.initialize(name, props); } @Override diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java index b08c640f..fe7020ba 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java @@ -40,7 +40,7 @@ public interface IcebergCatalogService { /** * Drop a namespace by first dropping all nested namespaces and tables underneath the namespace - * hierarchy. The empty namespace will not be dropped. + * hierarchy. The root namespace, {@link Namespace#empty()}, will not be dropped. * @param namespace the namespace to drop */ void dropNamespaceCascade(Namespace namespace); diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java index fb881579..0b5179af 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java @@ -48,6 +48,8 @@ public class PolarisApiService implements PolarisService { + private Map properties = null; + private String baseUrl = null; private PolarisManagementDefaultApi api = null; @@ -64,6 +66,8 @@ public PolarisApiService() {} @Override public void initialize(Map properties) throws Exception { + this.properties = properties; + String baseUrl = properties.get("base-url"); String token = properties.get("bearer-token"); @@ -269,7 +273,8 @@ private void setupOmnipotentCatalogRoleIfNotExists(String catalogName) { @Override public IcebergCatalogService initializeIcebergCatalogService(String catalogName) { setupOmnipotentCatalogRoleIfNotExists(catalogName); - return new PolarisIcebergCatalogService(baseUrl + "/api/catalog", catalogName, omnipotentPrincipal); + return new PolarisIcebergCatalogService( + baseUrl, catalogName, omnipotentPrincipal, properties); } } diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java index 9b1c3cb7..3955c676 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java @@ -23,6 +23,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.ResourcePaths; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog; import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService; @@ -36,13 +37,25 @@ public class PolarisIcebergCatalogService implements IcebergCatalogService { private final PolarisCatalog catalog; - public PolarisIcebergCatalogService(String uri, String catalogName, PrincipalWithCredentials migratorPrincipal) { + public PolarisIcebergCatalogService( + String baseUrl, + String catalogName, + PrincipalWithCredentials omnipotentPrincipal, + Map properties + ) { Map catalogProperties = new HashMap<>(); + String uri = baseUrl + "/api/catalog"; catalogProperties.put("uri", uri); catalogProperties.put("warehouse", catalogName); - String clientId = migratorPrincipal.getCredentials().getClientId(); - String clientSecret = migratorPrincipal.getCredentials().getClientSecret(); + // Default to /v1/oauth/tokens endpoint unless an explicit property was provided + String oauth2ServerUri = properties.getOrDefault( + "omnipotent-principal-oauth2-server-uri", uri + "/" + ResourcePaths.tokens()); + + catalogProperties.put("oauth2-server-uri", oauth2ServerUri); + + String clientId = omnipotentPrincipal.getCredentials().getClientId(); + String clientSecret = omnipotentPrincipal.getCredentials().getClientSecret(); catalogProperties.putIfAbsent( "credential", String.format("%s:%s", clientId, clientSecret)); catalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL"); diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 870a4b27..8023a504 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -60,7 +60,9 @@ public class SyncPolarisCommand implements Callable { "\nOmnipotent Principal Properties:" + "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + - "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the source Polaris" + + "\n\t- omnipotent-principal-oauth2-server-uri: (default: /v1/oauth/tokens endpoint for provided Polaris base-url) " + + "the OAuth2 server to use to authenticate the omnipotent-principal for Iceberg catalog access" ) private Map sourceProperties; @@ -79,7 +81,9 @@ public class SyncPolarisCommand implements Callable { "\nOmnipotent Principal Properties:" + "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + - "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the target Polaris" + + "\n\t- omnipotent-principal-oauth2-server-uri: (default: /v1/oauth/tokens endpoint for provided Polaris base-url) " + + "the OAuth2 server to use to retrieve a bearer token for the omnipotent-principal" ) private Map targetProperties; From 6f551b41c6b1b19f7c410c47eb70ea8b5efc54f0 Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Wed, 16 Apr 2025 17:44:39 -0700 Subject: [PATCH 17/18] Set iceberg write access as connection property explicitly outside of factory --- .../polaris/service/impl/PolarisApiService.java | 9 ++++++++- .../CreateOmnipotentPrincipalCommand.java | 10 +++------- .../sync/polaris/PolarisServiceFactory.java | 17 +++++++---------- .../tools/sync/polaris/SyncPolarisCommand.java | 12 ++++++++---- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java index 0b5179af..f454775e 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java @@ -48,6 +48,12 @@ public class PolarisApiService implements PolarisService { + /** + * Property that denotes whether this service should create omnipotent principals + * with write access when initializing {@link org.apache.iceberg.catalog.Catalog}. + */ + public static final String ICEBERG_WRITE_ACCESS_PROPERTY = "iceberg-write-access"; + private Map properties = null; private String baseUrl = null; @@ -99,7 +105,8 @@ public void initialize(Map properties) throws Exception { .clientSecret(properties.get("omnipotent-principal-client-secret"))); this.accessControlService = new AccessControlService(this); - this.icebergWriteAccess = Boolean.parseBoolean(properties.get("iceberg-write-access")); + this.icebergWriteAccess = Boolean.parseBoolean( + properties.getOrDefault(ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(false))); // by default false } @Override diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java index b6625de0..df65f81a 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java @@ -93,15 +93,11 @@ public class CreateOmnipotentPrincipalCommand implements Callable { @Override public Integer call() throws Exception { - polarisApiConnectionProperties.putIfAbsent("iceberg-write-access", String.valueOf(withWriteAccess)); + polarisApiConnectionProperties.putIfAbsent(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY, + String.valueOf(withWriteAccess)); PolarisService polaris = PolarisServiceFactory.createPolarisService( - PolarisServiceFactory.ServiceType.API, - withWriteAccess - ? PolarisServiceFactory.EndpointType.TARGET - : PolarisServiceFactory.EndpointType.SOURCE, - polarisApiConnectionProperties - ); + PolarisServiceFactory.ServiceType.API, polarisApiConnectionProperties); AccessControlService accessControlService = new AccessControlService((PolarisApiService) polaris); diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java index 26e06fd1..8d7ea28c 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java @@ -30,21 +30,18 @@ public enum ServiceType { API } - public enum EndpointType { - SOURCE, - TARGET - } - + /** + * Constructs and initializes a {@link PolarisService} implementation + * @param serviceType a recognized service type // TODO: support a custom implementation + * @param properties the properties to use to initialize the service + * @return the constructed and initialized service + */ public static PolarisService createPolarisService( ServiceType serviceType, - EndpointType endpointType, Map properties ) { PolarisService service = switch (serviceType) { - case API -> { - properties.putIfAbsent("iceberg-write-access", String.valueOf(endpointType == EndpointType.TARGET)); - yield new PolarisApiService(); - } + case API -> new PolarisApiService(); }; try { diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java index 8023a504..bfd64e25 100644 --- a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java +++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java @@ -28,6 +28,7 @@ import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner; import org.apache.polaris.tools.sync.polaris.service.PolarisService; +import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -125,11 +126,14 @@ public Integer call() throws Exception { SynchronizationPlanner modificationAwareSourceParityPlanner = new ModificationAwarePlanner(sourceParityPlanner); SynchronizationPlanner accessControlAwarePlanner = new AccessControlAwarePlanner(modificationAwareSourceParityPlanner); + // auto generate omnipotent principals with write access on the target, read only access on source + sourceProperties.put(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(false)); + targetProperties.put(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(true)); - PolarisService source = PolarisServiceFactory.createPolarisService( - PolarisServiceFactory.ServiceType.API, PolarisServiceFactory.EndpointType.SOURCE, sourceProperties); - PolarisService target = PolarisServiceFactory.createPolarisService( - PolarisServiceFactory.ServiceType.API, PolarisServiceFactory.EndpointType.TARGET, targetProperties); + PolarisService source = + PolarisServiceFactory.createPolarisService(PolarisServiceFactory.ServiceType.API, sourceProperties); + PolarisService target = + PolarisServiceFactory.createPolarisService(PolarisServiceFactory.ServiceType.API, targetProperties); ETagManager etagService = ETagManagerFactory.createETagManager(etagManagerType, etagManagerProperties); From 8cb745d94a9d472c744982ebb77b5e01277ddb0b Mon Sep 17 00:00:00 2001 From: mansehajsingh Date: Thu, 17 Apr 2025 09:25:57 -0700 Subject: [PATCH 18/18] Add external id to aws storage config ignore list --- .../tools/sync/polaris/planning/ModificationAwarePlanner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java index ae2bf4a8..15308abe 100644 --- a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java +++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java @@ -71,6 +71,7 @@ public class ModificationAwarePlanner implements SynchronizationPlanner { // S3 "storageConfigInfo.userArn", + "storageConfigInfo.externalId", // AZURE "storageConfigInfo.consentUrl",