diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts index 2476505406..4beccaa41e 100644 --- a/runtime/admin/build.gradle.kts +++ b/runtime/admin/build.gradle.kts @@ -32,6 +32,14 @@ dependencies { compileOnly("com.fasterxml.jackson.core:jackson-annotations") + implementation(project(":polaris-persistence-nosql-api")) + implementation(project(":polaris-persistence-nosql-maintenance-api")) + runtimeOnly(project(":polaris-persistence-nosql-metastore")) + runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus")) + runtimeOnly(project(":polaris-persistence-nosql-maintenance-impl")) + runtimeOnly(project(":polaris-persistence-nosql-metastore-maintenance")) + runtimeOnly(project(":polaris-persistence-nosql-authz-store-nosql")) + runtimeOnly(project(":polaris-relational-jdbc")) runtimeOnly("org.postgresql:postgresql") @@ -42,6 +50,9 @@ dependencies { implementation(project(":polaris-runtime-common")) + compileOnly("com.fasterxml.jackson.core:jackson-annotations") + compileOnly("com.fasterxml.jackson.core:jackson-databind") + testImplementation(project(":polaris-runtime-test-common")) testFixturesApi(project(":polaris-core")) @@ -52,6 +63,7 @@ dependencies { testFixturesApi(platform(libs.testcontainers.bom)) testFixturesApi("org.testcontainers:testcontainers") testFixturesApi("org.testcontainers:testcontainers-postgresql") + testFixturesImplementation(testFixtures(project(":polaris-persistence-nosql-mongodb"))) testRuntimeOnly("org.postgresql:postgresql") } diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java index 66ddaf0547..dcb427a8ec 100644 --- a/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java @@ -17,6 +17,7 @@ import io.quarkus.picocli.runtime.annotations.TopCommand; import java.io.PrintWriter; +import org.apache.polaris.admintool.nosql.NoSqlCommand; import org.apache.polaris.version.PolarisVersionProvider; import picocli.CommandLine.Command; import picocli.CommandLine.HelpCommand; @@ -31,8 +32,9 @@ HelpCommand.class, BootstrapCommand.class, PurgeCommand.class, + NoSqlCommand.class, }) -public class PolarisAdminTool extends BaseMetaStoreCommand { +public class PolarisAdminTool extends BaseCommand { @Override public Integer call() { diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/BaseNoSqlCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/BaseNoSqlCommand.java new file mode 100644 index 0000000000..8ed348be71 --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/BaseNoSqlCommand.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import jakarta.inject.Inject; +import org.apache.polaris.admintool.BaseCommand; +import org.apache.polaris.persistence.nosql.api.backend.Backend; + +public abstract class BaseNoSqlCommand extends BaseCommand { + @Inject protected Backend backend; + + protected void checkInMemory() { + if ("InMemory".equals(backend.type())) { + var err = spec.commandLine().getErr(); + + err.println(); + err.println("Running persistence-maintenance against InMemory is useless..."); + err.println(); + } + } + + protected void printNoSqlInfo() { + var out = spec.commandLine().getOut(); + + out.println("Polaris NoSql persistence has multiple subcommands,"); + out.println("use the 'help nosql' command."); + out.println(); + + checkInMemory(); + + out.println(); + out.println("Information: selected NoSql persistence backend: " + backend.type()); + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/NoSqlCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/NoSqlCommand.java new file mode 100644 index 0000000000..1dc9a52da6 --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/NoSqlCommand.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import jakarta.inject.Inject; +import org.apache.polaris.admintool.nosql.maintenance.NoSqlMaintenanceInfoCommand; +import org.apache.polaris.admintool.nosql.maintenance.NoSqlMaintenanceLogCommand; +import org.apache.polaris.admintool.nosql.maintenance.NoSqlMaintenanceRunCommand; +import org.apache.polaris.persistence.nosql.api.backend.Backend; +import picocli.CommandLine; + +@CommandLine.Command( + name = "nosql", + subcommands = { + NoSqlMaintenanceInfoCommand.class, + NoSqlMaintenanceLogCommand.class, + NoSqlMaintenanceRunCommand.class, + }, + mixinStandardHelpOptions = true, + description = "Polaris NoSQL persistence.") +public class NoSqlCommand extends BaseNoSqlCommand { + @Inject protected Backend backend; + + @Override + public Integer call() { + printNoSqlInfo(); + + return 0; + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/BaseNoSqlMaintenanceCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/BaseNoSqlMaintenanceCommand.java new file mode 100644 index 0000000000..39ab3bfc95 --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/BaseNoSqlMaintenanceCommand.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql.maintenance; + +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_COUNT_FROM_LAST_RUN_MULTIPLIER; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_CREATED_AT_GRACE_TIME; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_DELETE_BATCH_SIZE; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_EXPECTED_OBJ_COUNT; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_EXPECTED_REFERENCE_COUNT; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_INITIALIZED_FPP; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_MAX_ACCEPTABLE_FPP; +import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_RETAINED_RUNS; + +import jakarta.inject.Inject; +import java.io.PrintWriter; +import java.time.Instant; +import org.apache.polaris.admintool.nosql.BaseNoSqlCommand; +import org.apache.polaris.persistence.nosql.api.obj.ObjTypes; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunInformation; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunInformation.MaintenanceStats; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunSpec; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceService; + +@SuppressWarnings("CdiInjectionPointsInspection") +public abstract class BaseNoSqlMaintenanceCommand extends BaseNoSqlCommand { + @Inject protected MaintenanceService maintenanceService; + @Inject protected MaintenanceConfig maintenanceConfig; + + protected MaintenanceRunSpec printRealmStates() { + var out = spec.commandLine().getOut(); + + var runSpec = maintenanceService.buildMaintenanceRunSpec(); + out.println(); + out.printf("Process system realm: %s%n", runSpec.includeSystemRealm()); + out.println("Realms to process:"); + var realms = runSpec.realmsToProcess(); + if (realms.isEmpty()) { + out.println("(none)"); + } + for (var realm : realms) { + out.printf(" %s%n", realm); + } + + out.println("Realms to purge:"); + realms = runSpec.realmsToPurge(); + if (realms.isEmpty()) { + out.println("(none)"); + } + for (var realm : realms) { + out.printf(" %s%n", realm); + } + + return runSpec; + } + + protected void printMaintenanceConfig() { + var out = spec.commandLine().getOut(); + + out.println(); + out.println("Maintenance configuration:"); + out.printf( + " created-at grace time: %s%n", + maintenanceConfig.createdAtGraceTime().orElse(DEFAULT_CREATED_AT_GRACE_TIME)); + out.printf( + " delete batch size: %s%n", + maintenanceConfig.deleteBatchSize().orElse(DEFAULT_DELETE_BATCH_SIZE)); + out.printf( + " retained runs: %s%n", + maintenanceConfig.retainedRuns().orElse(DEFAULT_RETAINED_RUNS)); + + out.printf( + " expected object count: %d%n", + maintenanceConfig.expectedObjCount().orElse(DEFAULT_EXPECTED_OBJ_COUNT)); + out.printf( + " expected reference count: %d%n", + maintenanceConfig.expectedReferenceCount().orElse(DEFAULT_EXPECTED_REFERENCE_COUNT)); + out.printf( + " last-run multiplier: %f%n", + maintenanceConfig + .countFromLastRunMultiplier() + .orElse(DEFAULT_COUNT_FROM_LAST_RUN_MULTIPLIER)); + out.printf( + " initialized FPP: %f%n", + maintenanceConfig.filterInitializedFpp().orElse(DEFAULT_INITIALIZED_FPP)); + out.printf( + " expected FPP: %f%n", + maintenanceConfig.maxAcceptableFilterFpp().orElse(DEFAULT_MAX_ACCEPTABLE_FPP)); + + out.printf( + " reference scan rate limit / sec: %s%n", + maintenanceConfig.referenceScanRateLimitPerSecond().stream() + .mapToObj(Integer::toString) + .findFirst() + .orElse("(unlimited)")); + out.printf( + " object scan rate limit / sec: %s%n", + maintenanceConfig.objectScanRateLimitPerSecond().stream() + .mapToObj(Integer::toString) + .findFirst() + .orElse("(unlimited)")); + } + + protected void printRunInformation(MaintenanceRunInformation info, boolean expert) { + var out = spec.commandLine().getOut(); + out.println(); + out.println( + "=================================================================================="); + out.println(); + out.printf("Run started: %s%n", info.started()); + out.printf( + " status: %s%n", + info.statusMessage().orElse("(no exceptional information, all good so far)")); + out.printf(" finished: %s%n", info.finished().map(Instant::toString).orElse("(running)")); + out.printf(" details: %s%n", info.detailedInformation().orElse("-")); + + out.println(); + out.println("Realms:"); + out.printf(" purged: %d%n", info.purgedRealms().orElse(0)); + + out.println(); + out.println("References:"); + if (expert) { + // This is the number of calls to RetainedCollector.retainReference(), which is usually higher + // than the actual number of distinct reference names. + out.printf(" identified calls: %d%n", info.identifiedReferences().orElse(0)); + } + info.referenceStats().ifPresent(stats -> printStats(out, " ", stats)); + info.perRealmReferenceStats() + .forEach( + (realm, stats) -> { + out.printf(" Realm: %s%n", realm); + printStats(out, " ", stats); + }); + + out.println(); + out.println("Objects:"); + if (expert) { + // This is the number of calls to RetainedCollector.retainObj(), which is usually much higher + // than the actual number of distinct object references. + out.printf(" identified calls: %d%n", info.identifiedObjs().orElse(0)); + } + info.objStats().ifPresent(stats -> printStats(out, " ", stats)); + info.perRealmPerObjTypeStats() + .forEach( + (realm, perTypeStats) -> { + out.printf(" Realm: %s%n", realm); + perTypeStats.forEach( + (type, stats) -> { + out.printf(" Type: %s (%s)%n", type, ObjTypes.objTypeById(type).name()); + printStats(out, " ", stats); + }); + }); + } + + private void printStats(PrintWriter out, String indent, MaintenanceStats stats) { + out.printf("%s scanned: %d%n", indent, stats.scanned().orElse(0L)); + out.printf("%s retained: %d%n", indent, stats.retained().orElse(0L)); + out.printf("%s too new: %d%n", indent, stats.newer().orElse(0L)); + out.printf("%s purged: %d%n", indent, stats.purged().orElse(0L)); + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceInfoCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceInfoCommand.java new file mode 100644 index 0000000000..03d6c92aeb --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceInfoCommand.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql.maintenance; + +import picocli.CommandLine; + +@CommandLine.Command( + name = "maintenance-info", + mixinStandardHelpOptions = true, + description = "Polaris NoSQL persistence maintenance information.") +public class NoSqlMaintenanceInfoCommand extends BaseNoSqlMaintenanceCommand { + + @Override + public Integer call() { + printNoSqlInfo(); + + printMaintenanceConfig(); + + printRealmStates(); + + return 0; + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceLogCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceLogCommand.java new file mode 100644 index 0000000000..cae2a66905 --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceLogCommand.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql.maintenance; + +import picocli.CommandLine; + +@CommandLine.Command( + name = "maintenance-log", + mixinStandardHelpOptions = true, + description = "Show Polaris persistence maintenance log.") +public class NoSqlMaintenanceLogCommand extends BaseNoSqlMaintenanceCommand { + + @CommandLine.Option( + names = {"--show-expert"}, + description = "Show expert values, which only reflect internal operations.") + boolean showExpert; + + @Override + public Integer call() { + checkInMemory(); + + var infos = maintenanceService.maintenanceRunLog(); + var out = spec.commandLine().getOut(); + + out.println("Recorded Polaris NoSql persistence maintenance runs:"); + if (!infos.isEmpty()) { + infos.forEach(i -> printRunInformation(i, showExpert)); + } else { + out.println("(none)"); + } + + return 0; + } +} diff --git a/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceRunCommand.java b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceRunCommand.java new file mode 100644 index 0000000000..5c3fce09ba --- /dev/null +++ b/runtime/admin/src/main/java/org/apache/polaris/admintool/nosql/maintenance/NoSqlMaintenanceRunCommand.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql.maintenance; + +import jakarta.inject.Inject; +import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceService; +import picocli.CommandLine; + +@CommandLine.Command( + name = "maintenance-run", + mixinStandardHelpOptions = true, + description = {"Run Polaris persistence maintenance."}) +public class NoSqlMaintenanceRunCommand extends BaseNoSqlMaintenanceCommand { + @Inject MaintenanceService maintenanceService; + + // TODO once there's a fully-tested tasks "client" and 'MaintenanceTaskBehavior', _running_ + // maintenance should be directed through the tasks-API, giving users the option to run + // maintenance "locally" in the admin client or on any polaris server instance, while also + // ensuring (via the tasks framework) that only one maintenance run is active at any time. + + @Override + public Integer call() { + checkInMemory(); + + printMaintenanceConfig(); + var runSpec = printRealmStates(); + + var out = spec.commandLine().getOut(); + out.println(); + out.println("Starting NoSql persistence maintenance run..."); + out.println( + "This can run for quite some time, messages may be not be printed immediately, stay patient..."); + out.println(); + + var runInformation = maintenanceService.performMaintenance(runSpec); + + printRunInformation(runInformation, false); + + return 0; + } +} diff --git a/runtime/admin/src/main/resources/application.properties b/runtime/admin/src/main/resources/application.properties index 29229ebece..0de9f3fb16 100644 --- a/runtime/admin/src/main/resources/application.properties +++ b/runtime/admin/src/main/resources/application.properties @@ -36,11 +36,30 @@ quarkus.datasource.db-kind=postgresql # https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference for more details quarkus.rds.devservices.enabled=false quarkus.rds.sync-client.type=apache +quarkus.mongodb.devservices.enabled=false # ---- Runtime Configuration ---- # Below are default values for properties that can be changed in runtime. +# Available types: +# - in-memory - InMemoryPolarisMetaStoreManagerFactory +# - in-memory-atomic - InMemoryAtomicOperationMetaStoreManagerFactory +# - relational-jdbc - JdbcMetaStoreManagerFactory +# - nosql - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend' polaris.persistence.type=relational-jdbc +# Database backend for 'nosql' persistence-type +# Available backends: +# - InMemory - for testing purposes +# - MongoDb - configure the via the Quarkus extension +# Configure the necessary MongoDB properties starting with 'quarkus.mongodb.' in this file. +# See https://quarkus.io/guides/mongodb#configuration-reference for details about these configurations. +#polaris.persistence.nosql.backend=InMemory + +## MongoDB version store specific configuration +#quarkus.mongodb.database=polaris +#quarkus.mongodb.metrics.enabled=true +#quarkus.mongodb.connection-string=mongodb://localhost:27017 + #quarkus.datasource.active=false #quarkus.datasource.url= #quarkus.datasource.username= diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java new file mode 100644 index 0000000000..5f9d6f6c1f --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryBootstrapCommandTest.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.BootstrapCommandTestBase; + +@TestProfile(NoSqlInMemoryProfile.class) +class NoSqlInMemoryBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java new file mode 100644 index 0000000000..024631403d --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryProfile.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.Map; + +public class NoSqlInMemoryProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "InMemory"); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java new file mode 100644 index 0000000000..ecfacf5c31 --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlInMemoryPurgeCommandTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.TestProfile; +import java.util.Map; +import org.apache.polaris.admintool.PurgeCommandTestBase; + +@TestProfile(NoSqlInMemoryPurgeCommandTest.Profile.class) +class NoSqlInMemoryPurgeCommandTest extends PurgeCommandTestBase { + + public static class Profile extends NoSqlInMemoryProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("pre-bootstrap", "true") + .build(); + } + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java new file mode 100644 index 0000000000..22aa507b3d --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoBootstrapCommandTest.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.admintool.BootstrapCommandTestBase; + +@TestProfile(NoSqlMongoProfile.class) +class NoSqlMongoBootstrapCommandTest extends BootstrapCommandTestBase {} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java new file mode 100644 index 0000000000..595160853d --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoMaintenanceCommandTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.TestProfile; +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +@TestProfile(NoSqlMongoProfile.class) +@QuarkusMainTest +class NoSqlMongoMaintenanceCommandTest { + @Test + @Launch(value = {"help", "nosql"}) + void help(LaunchResult result) { + assertThat(result.getOutput()) + .contains("Commands:") + .contains("maintenance-log Show Polaris persistence maintenance log.") + .contains("maintenance-run Run Polaris persistence maintenance."); + } + + @Test + @Launch(value = {"nosql"}) + void noArg(LaunchResult result) { + assertThat(result.getOutput()) + .contains("Polaris NoSql persistence has multiple subcommands,") + .contains("use the 'help nosql' command.") + .contains("Information: selected NoSql persistence backend: MongoDb"); + } + + @Test + @Launch(value = {"nosql", "maintenance-info"}) + void info(LaunchResult result) { + assertThat(result.getOutput()) + .contains("Information: selected NoSql persistence backend: MongoDb") + .contains("Maintenance configuration:"); + } + + @Test + @Launch(value = {"nosql", "maintenance-log"}) + void logOfNothing(LaunchResult result) { + assertThat(result.getOutput()) + .contains("Recorded Polaris NoSql persistence maintenance runs:") + .contains("(none)"); + } + + @Test + @Launch(value = {"nosql", "maintenance-run"}) + void run(LaunchResult result) { + assertThat(result.getOutput()) + .contains("Maintenance configuration:") + .contains("Process system realm: true") + .contains("Starting NoSql persistence maintenance run...") + .contains("Run started: ") + .contains("status: (no exceptional information, all good so far)") + .contains("finished: 20") + .contains("References:") + .contains("Objects:") + .contains("Realm: ::system::"); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java new file mode 100644 index 0000000000..4b6f9959d1 --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoProfile.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; +import org.apache.polaris.admintool.MongoTestResourceLifecycleManager; + +public class NoSqlMongoProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.type", "nosql", "polaris.persistence.nosql.backend", "MongoDb"); + } + + @Override + public List testResources() { + return List.of(new TestResourceEntry(MongoTestResourceLifecycleManager.class)); + } +} diff --git a/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java new file mode 100644 index 0000000000..81deb680cc --- /dev/null +++ b/runtime/admin/src/test/java/org/apache/polaris/admintool/nosql/NoSqlMongoPurgeCommandTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool.nosql; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.TestProfile; +import java.util.Map; +import org.apache.polaris.admintool.PurgeCommandTestBase; + +@TestProfile(NoSqlMongoPurgeCommandTest.Profile.class) +class NoSqlMongoPurgeCommandTest extends PurgeCommandTestBase { + + public static class Profile extends NoSqlMongoProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("pre-bootstrap", "true") + .build(); + } + } +} diff --git a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/MongoTestResourceLifecycleManager.java b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/MongoTestResourceLifecycleManager.java new file mode 100644 index 0000000000..c42d47a20b --- /dev/null +++ b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/MongoTestResourceLifecycleManager.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.admintool; + +import io.quarkus.test.common.DevServicesContext; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; +import org.apache.polaris.persistence.nosql.mongodb.MongoDbBackendTestFactory; + +public class MongoTestResourceLifecycleManager + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + + private MongoDbBackendTestFactory mongoDbBackendTestFactory; + private DevServicesContext context; + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } + + @Override + @SuppressWarnings("resource") + public Map start() { + mongoDbBackendTestFactory = new MongoDbBackendTestFactory(); + mongoDbBackendTestFactory.start(context.containerNetworkId()); + return Map.of( + "quarkus.mongodb.connection-string", mongoDbBackendTestFactory.connectionString()); + } + + @Override + public void stop() { + if (mongoDbBackendTestFactory != null) { + try { + mongoDbBackendTestFactory.stop(); + } finally { + mongoDbBackendTestFactory = null; + } + } + } +}