Add REST Compatibility Kit

build.gradle

@@ -952,6 +952,51 @@ project(':iceberg-snowflake') {
 }
 
 project(':iceberg-open-api') {
+  apply plugin: 'java-test-fixtures'
+
+  dependencies {
+    testImplementation project(':iceberg-api')
+    testImplementation project(':iceberg-core')
+    testImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath
+    testImplementation(testFixtures(project(':iceberg-open-api')))
+
+    testImplementation libs.junit.jupiter
+    testImplementation libs.junit.suite.api
+    testImplementation libs.junit.suite.engine
+    testImplementation libs.assertj.core
+
+    testImplementation project(':iceberg-aws-bundle')
+    testImplementation project(':iceberg-gcp-bundle')
+    testImplementation project(':iceberg-azure-bundle')
+
+    testFixturesImplementation project(':iceberg-api')
+    testFixturesImplementation project(':iceberg-core')
+    testFixturesImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+    testFixturesImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath
+    testFixturesImplementation project(':iceberg-aws')
+    testFixturesImplementation project(':iceberg-gcp')
+    testFixturesImplementation project(':iceberg-azure')
+
+    testFixturesImplementation libs.jetty.servlet
+    testFixturesImplementation libs.jetty.server
+    testFixturesImplementation libs.sqlite.jdbc
+  }
+
+  test {
+    useJUnitPlatform()
+
+    // Always rerun the compatibility tests
+    outputs.upToDateWhen {false}
+    maxParallelForks = 1
+
+    // Pass through any system properties that start with "rck" (REST Compatibility Kit)
+    // Note: only pass through specific properties so they do not affect other build/test
+    //       configurations
+    systemProperties System.properties
+            .findAll { k, v -> k.startsWith("rck") }
+            .collectEntries { k, v -> { [(k):v, (k.replaceFirst("rck.", "")):v] }} // strip prefix
+  }
+
   def restCatalogSpec = "$projectDir/rest-catalog-open-api.yaml"
   tasks.register('validateRESTCatalogSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
     inputSpec.set(restCatalogSpec)

deploy.gradle

@@ -22,12 +22,8 @@ if (project.hasProperty('release') && jdkVersion != '11') {
 }
 
 subprojects {
-  if (it.name == 'iceberg-open-api') {
-    // don't publish iceberg-open-api
-    return
-  }
-
   def isBom = it.name == 'iceberg-bom'
+  def isOpenApi = it.name == 'iceberg-open-api'
 
   apply plugin: 'maven-publish'
   apply plugin: 'signing'
@@ -76,6 +72,9 @@ subprojects {
         apache(MavenPublication) {
           if (isBom) {
             from components.javaPlatform
+          } else if (isOpenApi) {
+            artifact testJar
+            artifact testFixturesJar
           } else {
             if (tasks.matching({task -> task.name == 'shadowJar'}).isEmpty()) {
               from components.java

gradle/libs.versions.toml

@@ -62,6 +62,7 @@ jaxb-api = "2.3.1"
 jaxb-runtime = "2.3.9"
 jetty = "11.0.22"
 junit = "5.10.1"
+junit-platform = "1.10.3"
 kafka = "3.8.0"
 kryo-shaded = "4.0.3"
 microprofile-openapi-api = "3.1.1"
@@ -202,6 +203,8 @@ jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty
 jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "jetty" }
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
+junit-suite-api = { module = "org.junit.platform:junit-platform-suite-api", version.ref = "junit-platform" }
+junit-suite-engine = { module = "org.junit.platform:junit-platform-suite-engine", version.ref = "junit-platform" }
 junit-vintage-engine = { module = "org.junit.vintage:junit-vintage-engine", version.ref = "junit" }
 kryo-shaded = { module = "com.esotericsoftware:kryo-shaded", version.ref = "kryo-shaded" }
 mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" }

open-api/README.md

@@ -40,3 +40,66 @@ make generate
 ```
 
 The generated code is not being used in the project, but helps to see what the changes in the open-API definition are in the generated code.
+
+# REST Compatibility Kit (RCK)
+
+The REST Compatibility Kit (RCK) is a Technology Compatibility Kit (TCK) implementation for the 
+Iceberg REST Specification.  This includes a series of tests based on the Java reference
+implementation of the REST Catalog that can be executed against any REST server that implements the
+spec.
+
+## Test Configuration
+
+The RCK can be configured using either environment variables or java system properties and allows
+for configuring both the tests and the REST client.  Environment variables prefixed by `CATALOG_`
+are passed through the catalog configuring with the following mutations:
+
+1. The `CATALOG_` prefix is stripped from the key name
+2. Single underscore (`_`) is replaced with a dot (`.`)
+3. Double underscore (`__`) is replaced with a dash (`-`)
+4. The key names are converted to lowercase
+
+A basic environment configuration would look like the following:
+
+```shell
+CATALOG_URI=https://my_rest_server.io/                    ## -> uri=https://my_rest_server.io/
+CATALOG_WAREHOUSE=test_warehouse                          ## -> warehouse=test_warehouse
+CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO       ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO
+CATALOG_CREDENTIAL=<oauth_key>:<oauth_secret>             ## -> credential=<oauth_key>:<oauth_secret>
+```
+
+Java properties passed to the test must be prefixed with `rck.`, which can be used to configure some
+test configurations described below and any catalog client properties.
+
+An example of the same configuration using java system properties would look like the following:
+```shell
+rck.uri=https://my_rest_server.io/                    ## -> uri=https://my_rest_server.io/
+rck.warehouse=test_warehouse                          ## -> warehouse=test_warehouse
+rck.io-impl=org.apache.iceberg.aws.s3.S3FileIO        ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO
+rck.credential=<oauth_key>:<oauth_secret>             ## -> credential=<oauth_key>:<oauth_secret>
+```
+
+Some test behaviors are configurable depending on the catalog implementations.  Not all behaviors
+are strictly defined by the REST Specification.  The following are currently configurable:
+
+| config                        | default |
+|-------------------------------|---------|
+| rck.requires-namespace-create | true    |
+| rck.supports-serverside-retry | true    |
+
+
+## Running Compatibility Tests
+
+The compatibility tests can be invoked via gradle with the following:
+
+Note:  The default behavior is to run a local http server with a jdbc backend for testing purposes,
+so `-Drck.local=false` must be set to point to an external REST server.
+
+```shell
+./gradlew :iceberg-open-api:test --tests RESTCompatibilityKitSuite \
+ -Drck.local=false \
+ -Drck.requires-namespace-create=true \
+ -Drck.uri=https://my_rest_server.io/ \
+ -Drck.warehouse=test_warehouse \
+ -Drck.credential=<oauth_key>:<oauth_secret>
+```

open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.util.PropertyUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@ExtendWith(RESTServerExtension.class)
+public class RESTCompatibilityKitCatalogTests extends CatalogTests<RESTCatalog> {
+  private static final Logger LOG = LoggerFactory.getLogger(RESTCompatibilityKitCatalogTests.class);
+
+  private static RESTCatalog restCatalog;
+
+  @BeforeAll
+  static void beforeClass() throws Exception {
+    restCatalog = RCKUtils.initCatalogClient();
+
+    assertThat(restCatalog.listNamespaces())
+        .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES)
+        .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES);
+  }
+
+  @BeforeEach
+  void before() {
+    try {
+      RCKUtils.purgeCatalogTestEntries(restCatalog);
+    } catch (Exception e) {
+      LOG.warn("Failure during test setup", e);
+    }
+  }
+
+  @AfterAll
+  static void afterClass() throws Exception {
+    restCatalog.close();
+  }
+
+  @Override
+  protected RESTCatalog catalog() {
+    return restCatalog;
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(),
+        RESTCompatibilityKitSuite.RCK_REQUIRES_NAMESPACE_CREATE,
+        super.requiresNamespaceCreate());
+  }
+
+  @Override
+  protected boolean supportsServerSideRetry() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_SERVERSIDE_RETRY, true);
+  }
+
+  @Override
+  protected boolean overridesRequestedLocation() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(),
+        RESTCompatibilityKitSuite.RCK_OVERRIDES_REQUESTED_LOCATION,
+        false);
+  }
+}

open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitSuite.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest;
+
+import org.junit.platform.suite.api.SelectClasses;
+import org.junit.platform.suite.api.Suite;
+import org.junit.platform.suite.api.SuiteDisplayName;
+
+/**
+ * Iceberg REST Compatibility Kit
+ *
+ * <p>This test suite provides the ability to run the Iceberg catalog tests against a remote REST
+ * catalog implementation to verify the behaviors against the reference implementation catalog
+ * tests.
+ *
+ * <p>The tests can be configured through environment variables or system properties. By default,
+ * the tests will run using a local http server using a servlet implementation that leverages the
+ * {@link RESTCatalogAdapter}.
+ */
+@Suite
+@SuiteDisplayName("Iceberg REST Compatibility Kit")
+@SelectClasses({RESTCompatibilityKitCatalogTests.class, RESTCompatibilityKitViewCatalogTests.class})
+public class RESTCompatibilityKitSuite {
+  static final String RCK_REQUIRES_NAMESPACE_CREATE = "rck.requires-namespace-create";
+  static final String RCK_SUPPORTS_SERVERSIDE_RETRY = "rck.supports-serverside-retry";
+  static final String RCK_OVERRIDES_REQUESTED_LOCATION = "rck.overrides-requested-location";
+
+  protected RESTCompatibilityKitSuite() {}
+}

open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitViewCatalogTests.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.view.ViewCatalogTests;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@ExtendWith(RESTServerExtension.class)
+public class RESTCompatibilityKitViewCatalogTests extends ViewCatalogTests<RESTCatalog> {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RESTCompatibilityKitViewCatalogTests.class);
+  private static RESTCatalog restCatalog;
+
+  @BeforeAll
+  static void beforeClass() throws Exception {
+    restCatalog = RCKUtils.initCatalogClient();
+
+    assertThat(restCatalog.listNamespaces())
+        .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES)
+        .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES);
+  }
+
+  @BeforeEach
+  void before() {
+    try {
+      RCKUtils.purgeCatalogTestEntries(restCatalog);
+    } catch (Exception e) {
+      LOG.warn("Failure during test setup", e);
+    }
+  }
+
+  @AfterAll
+  static void afterClass() throws Exception {
+    restCatalog.close();
+  }
+
+  @Override
+  protected RESTCatalog catalog() {
+    return restCatalog;
+  }
+
+  @Override
+  protected Catalog tableCatalog() {
+    return restCatalog;
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(), RESTCompatibilityKitSuite.RCK_REQUIRES_NAMESPACE_CREATE, true);
+  }
+
+  @Override
+  protected boolean supportsServerSideRetry() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_SERVERSIDE_RETRY, true);
+  }
+
+  @Override
+  protected boolean overridesRequestedLocation() {
+    return PropertyUtil.propertyAsBoolean(
+        restCatalog.properties(),
+        RESTCompatibilityKitSuite.RCK_OVERRIDES_REQUESTED_LOCATION,
+        false);
+  }
+}