Implement GenericTableCatalogAdapter; admin-related fixes

integration-tests/build.gradle.kts

@@ -22,6 +22,7 @@ plugins { id("polaris-server") }
 dependencies {
   implementation(project(":polaris-core"))
   implementation(project(":polaris-api-management-model"))
+  implementation(project(":polaris-api-catalog-service"))
 
   implementation(libs.jakarta.ws.rs.api)
   implementation(libs.guava)

integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.env;
+
+import static jakarta.ws.rs.core.Response.Status.NO_CONTENT;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.core.Response;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.rest.RESTUtil;
+import org.apache.polaris.service.types.CreateGenericTableRequest;
+import org.apache.polaris.service.types.GenericTable;
+import org.apache.polaris.service.types.ListGenericTablesResponse;
+import org.apache.polaris.service.types.LoadGenericTableResponse;
+
+/**
+ * A simple, non-exhaustive set of helper methods for accessing the generic tables REST API
+ *
+ * @see PolarisClient#genericTableApi(ClientCredentials)
+ */
+public class GenericTableApi extends RestApi {
+  GenericTableApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) {
+    super(client, endpoints, authToken, uri);
+  }
+
+  public void purge(String catalog, Namespace ns) {
+    listGenericTables(catalog, ns).forEach(t -> dropGenericTable(catalog, t));
+  }
+
+  public List<TableIdentifier> listGenericTables(String catalog, Namespace namespace) {
+    String ns = RESTUtil.encodeNamespace(namespace);
+    try (Response res =
+        request("polaris/v1/{cat}/namespaces/{ns}/generic-tables", Map.of("cat", catalog, "ns", ns))
+            .get()) {
+      assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode());
+      return res.readEntity(ListGenericTablesResponse.class).getIdentifiers().stream().toList();
+    }
+  }
+
+  public void dropGenericTable(String catalog, TableIdentifier id) {
+    String ns = RESTUtil.encodeNamespace(id.namespace());
+    try (Response res =
+        request(
+                "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}",
+                Map.of("cat", catalog, "table", id.name(), "ns", ns))
+            .delete()) {
+      assertThat(res.getStatus()).isEqualTo(NO_CONTENT.getStatusCode());
+    }
+  }
+
+  public GenericTable getGenericTable(String catalog, TableIdentifier id) {
+    String ns = RESTUtil.encodeNamespace(id.namespace());
+    try (Response res =
+        request(
+                "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}",
+                Map.of("cat", catalog, "table", id.name(), "ns", ns))
+            .get()) {
+      return res.readEntity(LoadGenericTableResponse.class).getTable();
+    }
+  }
+
+  public GenericTable createGenericTable(
+      String catalog, TableIdentifier id, String format, Map<String, String> properties) {
+    String ns = RESTUtil.encodeNamespace(id.namespace());
+    try (Response res =
+        request(
+                "polaris/v1/{cat}/namespaces/{ns}/generic-tables/",
+                Map.of("cat", catalog, "ns", ns))
+            .post(
+                Entity.json(new CreateGenericTableRequest(id.name(), format, "doc", properties)))) {
+      return res.readEntity(LoadGenericTableResponse.class).getTable();
+    }
+  }
+}

integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java

@@ -106,6 +106,16 @@ public CatalogApi catalogApiPlain() {
     return new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint());
   }
 
+  public GenericTableApi genericTableApi(PrincipalWithCredentials principal) {
+    return new GenericTableApi(
+        client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint());
+  }
+
+  public GenericTableApi genericTableApi(ClientCredentials credentials) {
+    return new GenericTableApi(
+        client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint());
+  }
+
   /**
    * Requests an access token from the Polaris server for the client ID/secret pair that is part of
    * the given principal data object.

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java

@@ -18,11 +18,13 @@
  */
 package org.apache.polaris.service.it.test;
 
+import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
 import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import com.google.common.collect.ImmutableMap;
+import jakarta.ws.rs.ProcessingException;
 import jakarta.ws.rs.client.Entity;
 import jakarta.ws.rs.client.Invocation;
 import jakarta.ws.rs.core.HttpHeaders;
@@ -32,11 +34,13 @@
 import java.lang.reflect.Method;
 import java.net.URI;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Stream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.iceberg.BaseTable;
 import org.apache.iceberg.BaseTransaction;
@@ -57,6 +61,7 @@
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.io.ResolvingFileIO;
 import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.iceberg.rest.RESTUtil;
 import org.apache.iceberg.rest.requests.CreateTableRequest;
 import org.apache.iceberg.rest.responses.ErrorResponse;
 import org.apache.iceberg.types.Types;
@@ -82,12 +87,14 @@
 import org.apache.polaris.core.entity.PolarisEntityConstants;
 import org.apache.polaris.service.it.env.CatalogApi;
 import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.GenericTableApi;
 import org.apache.polaris.service.it.env.IcebergHelper;
 import org.apache.polaris.service.it.env.IntegrationTestsHelper;
 import org.apache.polaris.service.it.env.ManagementApi;
 import org.apache.polaris.service.it.env.PolarisApiEndpoints;
 import org.apache.polaris.service.it.env.PolarisClient;
 import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
+import org.apache.polaris.service.types.GenericTable;
 import org.assertj.core.api.Assertions;
 import org.assertj.core.api.InstanceOfAssertFactories;
 import org.junit.jupiter.api.AfterAll;
@@ -127,6 +134,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
 
   private PrincipalWithCredentials principalCredentials;
   private CatalogApi catalogApi;
+  private GenericTableApi genericTableApi;
   private RESTCatalog restCatalog;
   private String currentCatalogName;
   private Map<String, String> restCatalogConfig;
@@ -178,6 +186,7 @@ public void before(TestInfo testInfo) {
     principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
 
     catalogApi = client.catalogApi(principalCredentials);
+    genericTableApi = client.genericTableApi(principalCredentials);
 
     Method method = testInfo.getTestMethod().orElseThrow();
     currentCatalogName = client.newEntityName(method.getName());
@@ -1194,4 +1203,143 @@ public void testLoadCredentials() {
       assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
     }
   }
+
+  @Test
+  public void testCreateGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1");
+
+    GenericTable createResponse =
+        genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of());
+    Assertions.assertThat(createResponse.getFormat()).isEqualTo("format");
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testLoadGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1");
+
+    genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of());
+
+    GenericTable loadResponse =
+        genericTableApi.getGenericTable(currentCatalogName, tableIdentifier);
+    Assertions.assertThat(loadResponse.getFormat()).isEqualTo("format");
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testListGenericTables() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier1 = TableIdentifier.of(namespace, "tbl1");
+    TableIdentifier tableIdentifier2 = TableIdentifier.of(namespace, "tbl2");
+
+    genericTableApi.createGenericTable(currentCatalogName, tableIdentifier1, "format", Map.of());
+    genericTableApi.createGenericTable(currentCatalogName, tableIdentifier2, "format", Map.of());
+
+    List<TableIdentifier> identifiers =
+        genericTableApi.listGenericTables(currentCatalogName, namespace);
+
+    Assertions.assertThat(identifiers).hasSize(2);
+    Assertions.assertThat(identifiers)
+        .containsExactlyInAnyOrder(tableIdentifier1, tableIdentifier2);
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testDropGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1");
+
+    genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of());
+
+    GenericTable loadResponse =
+        genericTableApi.getGenericTable(currentCatalogName, tableIdentifier);
+    Assertions.assertThat(loadResponse.getFormat()).isEqualTo("format");
+
+    genericTableApi.dropGenericTable(currentCatalogName, tableIdentifier);
+
+    Assertions.assertThatCode(
+            () -> genericTableApi.getGenericTable(currentCatalogName, tableIdentifier))
+        .isInstanceOf(ProcessingException.class);
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testGrantsOnGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1");
+    genericTableApi.createGenericTable(currentCatalogName, tableIdentifier, "format", Map.of());
+
+    managementApi.createCatalogRole(currentCatalogName, "catalogrole1");
+
+    Stream<TableGrant> tableGrants =
+        Arrays.stream(TablePrivilege.values())
+            .map(
+                p -> {
+                  return new TableGrant(List.of("ns1"), "tbl1", p, GrantResource.TypeEnum.TABLE);
+                });
+
+    tableGrants.forEach(g -> managementApi.addGrant(currentCatalogName, "catalogrole1", g));
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testGrantsOnNonExistingGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+
+    managementApi.createCatalogRole(currentCatalogName, "catalogrole1");
+
+    Stream<TableGrant> tableGrants =
+        Arrays.stream(TablePrivilege.values())
+            .map(
+                p -> {
+                  return new TableGrant(List.of("ns1"), "tbl1", p, GrantResource.TypeEnum.TABLE);
+                });
+
+    tableGrants.forEach(
+        g -> {
+          try (Response response =
+              managementApi
+                  .request(
+                      "v1/catalogs/{cat}/catalog-roles/{role}/grants",
+                      Map.of("cat", currentCatalogName, "role", "catalogrole1"))
+                  .put(Entity.json(g))) {
+
+            assertThat(response.getStatus()).isEqualTo(NOT_FOUND.getStatusCode());
+          }
+        });
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
+
+  @Test
+  public void testDropNonExistingGenericTable() {
+    Namespace namespace = Namespace.of("ns1");
+    restCatalog.createNamespace(namespace);
+    TableIdentifier tableIdentifier = TableIdentifier.of(namespace, "tbl1");
+
+    String ns = RESTUtil.encodeNamespace(tableIdentifier.namespace());
+    try (Response res =
+        genericTableApi
+            .request(
+                "polaris/v1/{cat}/namespaces/{ns}/generic-tables/{table}",
+                Map.of("cat", currentCatalogName, "table", tableIdentifier.name(), "ns", ns))
+            .delete()) {
+      assertThat(res.getStatus()).isEqualTo(NOT_FOUND.getStatusCode());
+    }
+
+    genericTableApi.purge(currentCatalogName, namespace);
+  }
 }

polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java

@@ -188,6 +188,6 @@ protected FeatureConfiguration(
       PolarisConfiguration.<Boolean>builder()
           .key("ENABLE_GENERIC_TABLES")
           .description("If true, the generic-tables endpoints are enabled")
-          .defaultValue(false)
+          .defaultValue(true)
           .buildFeatureConfiguration();
 }

regtests/t_spark_sql/ref/spark_sql_basic.sh.ref

@@ -1,4 +1,4 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]}
+{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit","GET polaris/v1/{prefix}/namespaces/{namespace}/generic-tables","POST polaris/v1/{prefix}/namespaces/{namespace}/generic-tables","DELETE polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}","GET polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}"]}
 Catalog created
 spark-sql (default)> use polaris;
 spark-sql ()> show namespaces;

regtests/t_spark_sql/ref/spark_sql_views.sh.ref

@@ -1,4 +1,4 @@
-{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]}
+{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit","GET polaris/v1/{prefix}/namespaces/{namespace}/generic-tables","POST polaris/v1/{prefix}/namespaces/{namespace}/generic-tables","DELETE polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}","GET polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}"]}
 Catalog created
 spark-sql (default)> use polaris;
 spark-sql ()> show namespaces;