NoSQL: Add to runtime-service

runtime/defaults/src/main/resources/application-test.properties

@@ -22,6 +22,7 @@
 
 quarkus.datasource.devservices.enabled=false
 quarkus.keycloak.devservices.enabled=false
+quarkus.mongodb.devservices.enabled=false
 
 quarkus.log.level=ERROR
 quarkus.log.file.enable=false

runtime/defaults/src/main/resources/application.properties

@@ -41,6 +41,8 @@ quarkus.micrometer.enabled=true
 quarkus.micrometer.export.prometheus.enabled=true
 quarkus.oidc.enabled=true
 quarkus.otel.enabled=true
+#quarkus.mongodb.metrics.enabled=true
+#quarkus.mongodb.connection-string=mongodb://localhost:27017
 
 # ---- Runtime Configuration ----
 # Below are default values for properties that can be changed in runtime.
@@ -128,16 +130,29 @@ polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"]
 # realm overrides
 # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true
 
-# polaris.persistence.type=in-memory-atomic
+# Available types:
+# - in-memory - InMemoryPolarisMetaStoreManagerFactory
+# - in-memory-atomic - InMemoryAtomicOperationMetaStoreManagerFactory
+# - nosql (beta) - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend'
+# - relational-jdbc
 polaris.persistence.type=in-memory
-# polaris.persistence.type=relational-jdbc
+# Database backend for 'nosql' persistence-type
+# Available backends:
+# - InMemory - for testing purposes
+# - MongoDb - configure the via the Quarkus extension
+#     Configure the necessary MongoDB properties starting with 'quarkus.mongodb.' in this file.
+#     See https://quarkus.io/guides/mongodb#configuration-reference for details about these configurations.
+#polaris.persistence.nosql.backend=InMemory
 
 polaris.secrets-manager.type=in-memory
 # if set to true it will try to start localstack at build and run time for the local environment
 # https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference for more details
 quarkus.rds.devservices.enabled=false
 quarkus.rds.sync-client.type=apache
 
+## MongoDB specific configuration
+#quarkus.mongodb.database=polaris
+
 polaris.file-io.type=default
 
 polaris.event-listener.type=no-op
@@ -283,6 +298,8 @@ quarkus.arc.ignored-split-packages=\
 
 ## Quarkus required setting for third party indexing
 # fixed at build-time
+quarkus.index-dependency.agrona.group-id=org.agrona
+quarkus.index-dependency.agrona.artifact-id=agrona
 quarkus.index-dependency.avro.group-id=org.apache.avro
 quarkus.index-dependency.avro.artifact-id=avro
 quarkus.index-dependency.guava.group-id=com.google.guava

runtime/service/build.gradle.kts

@@ -39,6 +39,12 @@ dependencies {
   compileOnly(project(":polaris-immutables"))
   annotationProcessor(project(":polaris-immutables", configuration = "processor"))
 
+  runtimeOnly(project(":polaris-persistence-nosql-metastore"))
+  runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus"))
+  runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus-distcache"))
+  runtimeOnly(project(":polaris-persistence-nosql-maintenance-impl"))
+  runtimeOnly(project(":polaris-persistence-nosql-metastore-maintenance"))
+
   implementation(platform(libs.iceberg.bom))
   implementation("org.apache.iceberg:iceberg-api")
   implementation("org.apache.iceberg:iceberg-core")
@@ -153,6 +159,10 @@ dependencies {
   testImplementation("org.testcontainers:testcontainers")
   testImplementation("org.testcontainers:testcontainers-postgresql")
 
+  testImplementation(project(":polaris-persistence-nosql-api"))
+  testImplementation(testFixtures(project(":polaris-persistence-nosql-api")))
+  testImplementation(project(":polaris-persistence-nosql-impl"))
+
   testFixturesImplementation(project(":polaris-core"))
   testFixturesImplementation(project(":polaris-api-management-model"))
   testFixturesImplementation(project(":polaris-api-management-service"))

runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlApplicationIT.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.nosql;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest;
+
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlTesting.PersistenceInMemoryProfile.class)
+public class NoSqlApplicationIT extends PolarisApplicationIntegrationTest {}

runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlCatalogIT.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.nosql;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import java.util.Map;
+import org.apache.polaris.service.it.PolarisRestCatalogMinIOIT;
+
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlCatalogIT.Profile.class)
+public class NoSqlCatalogIT extends PolarisRestCatalogMinIOIT {
+  public static class Profile extends NoSqlTesting.PersistenceInMemoryProfile {
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return ImmutableMap.<String, String>builder()
+          .putAll(super.getConfigOverrides())
+          .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY)
+          .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY)
+          .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false")
+          .build();
+    }
+  }
+}

runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlManagementServiceIT.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.nosql;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest;
+
+@QuarkusIntegrationTest
+@TestProfile(value = NoSqlTesting.PersistenceInMemoryProfile.class)
+public class NoSqlManagementServiceIT extends PolarisManagementServiceIntegrationTest {}

runtime/service/src/intTest/java/org/apache/polaris/service/it/nosql/NoSqlTesting.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.nosql;
+
+import io.quarkus.test.junit.QuarkusTestProfile;
+import java.util.Map;
+
+public final class NoSqlTesting {
+  private NoSqlTesting() {}
+
+  public static class PersistenceInMemoryProfile implements QuarkusTestProfile {
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return Map.of(
+          "polaris.persistence.type",
+          "nosql",
+          "polaris.persistence.nosql.backend",
+          "InMemory",
+          "polaris.persistence.auto-bootstrap-types",
+          "nosql");
+    }
+  }
+}

runtime/service/src/test/java/org/apache/polaris/service/catalog/Profiles.java

@@ -19,12 +19,19 @@
 
 package org.apache.polaris.service.catalog;
 
+import com.google.common.collect.ImmutableMap;
 import io.quarkus.test.junit.QuarkusTestProfile;
 import java.util.Map;
 
 public final class Profiles {
   private Profiles() {}
 
+  public static final Map<String, String> NOSQL_IN_MEM =
+      ImmutableMap.<String, String>builder()
+          .put("polaris.persistence.type", "nosql")
+          .put("polaris.persistence.nosql.backend", "InMemory")
+          .build();
+
   public static class DefaultProfile implements QuarkusTestProfile {
     @Override
     public Map<String, String> getConfigOverrides() {
@@ -41,4 +48,14 @@ public Map<String, String> getConfigOverrides() {
           "true");
     }
   }
+
+  public static class DefaultNoSqlProfile extends DefaultProfile {
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return ImmutableMap.<String, String>builder()
+          .putAll(super.getConfigOverrides())
+          .putAll(NOSQL_IN_MEM)
+          .build();
+    }
+  }
 }

runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalogNoSqlInMemTest.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.generic;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.service.catalog.Profiles;
+
+@QuarkusTest
+@TestProfile(Profiles.DefaultNoSqlProfile.class)
+public class PolarisGenericTableCatalogNoSqlInMemTest
+    extends AbstractPolarisGenericTableCatalogTest {
+
+  @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+  @Override
+  protected void bootstrapRealm(String realmName) {
+    metaStoreManagerFactory
+        .bootstrapRealms(
+            List.of(realmName),
+            RootCredentialsSet.fromList(List.of(realmName + ",aClientId,aSecret")))
+        .get(realmName);
+  }
+}

runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerNoSqlAuthzTest.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.apache.polaris.service.catalog.Profiles.NOSQL_IN_MEM;
+
+import com.google.common.collect.ImmutableMap;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import java.util.List;
+import java.util.Map;
+import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
+import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet;
+import org.apache.polaris.service.admin.PolarisAuthzTestBase;
+
+@QuarkusTest
+@TestProfile(IcebergCatalogHandlerNoSqlAuthzTest.Profile.class)
+public class IcebergCatalogHandlerNoSqlAuthzTest extends AbstractIcebergCatalogHandlerAuthzTest {
+
+  public static class Profile extends PolarisAuthzTestBase.Profile {
+    @Override
+    public Map<String, String> getConfigOverrides() {
+      return ImmutableMap.<String, String>builder()
+          .putAll(super.getConfigOverrides())
+          .putAll(NOSQL_IN_MEM)
+          .build();
+    }
+  }
+
+  @Inject MetaStoreManagerFactory metaStoreManagerFactory;
+
+  @Override
+  protected void bootstrapRealm(String realmIdentifier) {
+    metaStoreManagerFactory.bootstrapRealms(List.of(realmIdentifier), RootCredentialsSet.EMPTY);
+  }
+}