OpenHouse <-> Iceberg Compatibiltiy Test Infra

api/src/test/java/org/apache/iceberg/ParameterizedTestExtension.java

@@ -20,7 +20,9 @@
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.text.MessageFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -70,18 +72,16 @@ public boolean supportsTestTemplate(ExtensionContext context) {
   public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(
       ExtensionContext context) {
 
-    // Search method annotated with @Parameters
+    // Search for methods annotated with @Parameters, preferring the most specific class
     final List<Method> parameterProviders =
         AnnotationSupport.findAnnotatedMethods(
             context.getRequiredTestClass(), Parameters.class, HierarchyTraversalMode.TOP_DOWN);
-    if (parameterProviders.isEmpty()) {
+    Method parameterProvider =
+        resolveParameterProvider(context.getRequiredTestClass(), parameterProviders);
+    if (parameterProvider == null) {
       throw new IllegalStateException("Cannot find any parameter provider");
     }
-    if (parameterProviders.size() > 1) {
-      throw new IllegalStateException("Multiple parameter providers are found");
-    }
 
-    Method parameterProvider = parameterProviders.get(0);
     // Get potential test name
     String testNameTemplate = parameterProvider.getAnnotation(Parameters.class).name();
 
@@ -97,18 +97,30 @@ public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContex
 
     Preconditions.checkState(parameterValues != null, "Parameter values cannot be null");
 
-    // Parameter values could be Object[][]
-    if (parameterValues instanceof Object[][]) {
-      Object[][] typedParameterValues = (Object[][]) parameterValues;
-      return createContextForParameters(
-          Arrays.stream(typedParameterValues), testNameTemplate, context);
+    List<Object[]> allParameters = new ArrayList<>();
+    normalizeParameters(parameterValues).forEach(allParameters::add);
+
+    try {
+      Method extraMethod = context.getRequiredTestClass().getMethod("getExtraParameters");
+      if (Modifier.isStatic(extraMethod.getModifiers())) {
+        Object extra = extraMethod.invoke(null);
+        normalizeParameters(extra).forEach(allParameters::add);
+      }
+    } catch (NoSuchMethodException e) {
+      // ignore
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to invoke getExtraParameters", e);
     }
 
-    // or a Collection
-    if (parameterValues instanceof Collection) {
-      final Collection<?> typedParameterValues = (Collection<?>) parameterValues;
-      final Stream<Object[]> parameterValueStream =
-          typedParameterValues.stream()
+    return createContextForParameters(allParameters.stream(), testNameTemplate, context);
+  }
+
+  private Stream<Object[]> normalizeParameters(Object parameters) {
+    if (parameters instanceof Object[][]) {
+      return Arrays.stream((Object[][]) parameters);
+    } else if (parameters instanceof Collection) {
+      return ((Collection<?>) parameters)
+          .stream()
               .map(
                   (Function<Object, Object[]>)
                       parameterValue -> {
@@ -118,13 +130,36 @@ public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContex
                           return new Object[] {parameterValue};
                         }
                       });
-      return createContextForParameters(parameterValueStream, testNameTemplate, context);
     }
-
     throw new IllegalStateException(
         String.format(
-            "Return type of @Parameters annotated method \"%s\" should be either Object[][] or Collection",
-            parameterProvider));
+            "Return type of @Parameters annotated method should be either Object[][] or Collection, but was %s",
+            parameters.getClass().getName()));
+  }
+
+  /**
+   * Resolves the parameter provider method, preferring the most specific (child) class when multiple
+   * @Parameters methods exist in the hierarchy.
+   */
+  private static Method resolveParameterProvider(
+      Class<?> testClass, List<Method> parameterProviders) {
+    if (parameterProviders.isEmpty()) {
+      return null;
+    }
+    if (parameterProviders.size() == 1) {
+      return parameterProviders.get(0);
+    }
+
+    // Walk up the hierarchy, return the first match (most specific class)
+    for (Class<?> current = testClass; current != null; current = current.getSuperclass()) {
+      for (Method candidate : parameterProviders) {
+        if (candidate.getDeclaringClass().equals(current)) {
+          return candidate;
+        }
+      }
+    }
+
+    return parameterProviders.get(0);
   }
 
   private static class FieldInjectingInvocationContext implements TestTemplateInvocationContext {

build.gradle

@@ -48,6 +48,12 @@ buildscript {
 String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
 String sparkVersionsString = System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")
 List<String> sparkVersions = sparkVersionsString != null && !sparkVersionsString.isEmpty() ? sparkVersionsString.split(",") : []
+// OpenHouse compatibility testing configuration
+String openHouseCompatibilitySparkMajorVersion = findProperty("openhouseCompatibilitySparkMajorVersion") ?: '3.5'
+ext {
+  openHouseCompatibilityCoordinate = findProperty("openhouseCompatibilityCoordinate") ?:
+      'com.linkedin.openhouse:tables-test-fixtures-iceberg-1.5_2.12:0.0.+:uber'
+}
 
 try {
   // apply these plugins in a try-catch block so that we can handle cases without .git directory
@@ -128,6 +134,10 @@ allprojects {
   repositories {
     mavenCentral()
     mavenLocal()
+    // LinkedIn OpenHouse artifacts for compatibility testing
+    maven {
+      url "https://linkedin.jfrog.io/artifactory/openhouse"
+    }
   }
 }
 
@@ -335,9 +345,17 @@ project(':iceberg-common') {
 }
 
 project(':iceberg-core') {
+  configurations {
+    openhouseCompatibilityRuntime {
+      canBeConsumed = false
+      canBeResolved = true
+    }
+  }
+
   test {
     useJUnitPlatform()
   }
+
   dependencies {
     api project(':iceberg-api')
     implementation project(':iceberg-common')
@@ -370,6 +388,63 @@ project(':iceberg-core') {
     testImplementation libs.esotericsoftware.kryo
     testImplementation libs.guava.testlib
     testImplementation libs.awaitility
+
+    openhouseCompatibilityRuntime(rootProject.openHouseCompatibilityCoordinate) {
+      transitive = false
+    }
+    openhouseCompatibilityRuntime project(':iceberg-aws')
+    openhouseCompatibilityRuntime 'com.google.code.gson:gson:2.10.1'
+    openhouseCompatibilityRuntime 'com.zaxxer:HikariCP:4.0.3'
+  }
+
+  tasks.register('openhouseCompatibilityTest', Test) {
+    useJUnitPlatform()
+    group = 'verification'
+    description = 'Runs OpenHouse compatibility tests for iceberg-core'
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath + configurations.openhouseCompatibilityRuntime
+
+    systemProperty 'iceberg.test.table.provider',
+        'org.apache.iceberg.openhouse.OpenHouseTestTableProvider'
+
+    // Test filter - initially exclude everything, then include from list
+    filter {
+      failOnNoMatchingTests = false
+    }
+
+    // Load inclusions from the fixtures jar at execution time
+    doFirst {
+      println "DEBUG: Resolving openhouseCompatibilityRuntime..."
+      configurations.openhouseCompatibilityRuntime.resolve().each { file ->
+        if (file.name.contains('tables-test-fixtures')) {
+          println "DEBUG: Found fixtures jar: ${file.name}"
+          zipTree(file).matching { include 'openhouse-iceberg-compatibility-tests.txt' }.each { f ->
+            println "DEBUG: Found inclusions file: ${f}"
+            f.eachLine { line ->
+              line = line.trim()
+              if (line && !line.startsWith('#')) {
+                // Replace # with . for Gradle test filter pattern
+                line = line.replace('#', '.')
+
+                // Append wildcard to handle parameterized tests
+                if (!line.endsWith('*')) {
+                  line = line + '*'
+                }
+                println "DEBUG: Including ${line}"
+                filter.includeTestsMatching line
+              }
+            }
+          }
+        }
+      }
+    }
+
+
+    testLogging {
+      events "passed", "skipped", "failed", "standardOut", "standardError"
+      showStandardStreams = true
+      exceptionFormat = 'full'
+    }
   }
 }
 
@@ -1030,6 +1105,14 @@ apply from: 'baseline.gradle'
 apply from: 'deploy.gradle'
 apply from: 'tasks.gradle'
 
+tasks.register('openhouseCompatibility') {
+  group = "verification"
+  description = "Runs all OpenHouse compatibility tests"
+  dependsOn project(':iceberg-core').tasks.openhouseCompatibilityTest
+  dependsOn project(":iceberg-spark:iceberg-spark-${openHouseCompatibilitySparkMajorVersion}_${scalaVersion}").tasks.openhouseCompatibilityTest
+  dependsOn project(":iceberg-spark:iceberg-spark-extensions-${openHouseCompatibilitySparkMajorVersion}_${scalaVersion}").tasks.openhouseCompatibilityTest
+}
+
 project(':iceberg-bom') {
   apply plugin: 'java-platform'
 

core/src/test/java/org/apache/iceberg/TableTestBase.java

@@ -43,6 +43,7 @@
 import org.apache.iceberg.types.Conversions;
 import org.apache.iceberg.types.Types;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -171,6 +172,10 @@ public class TableTestBase {
   protected File metadataDir = null;
   public TestTables.TestTable table = null;
 
+  // External table provider loaded via reflection to avoid compile-time dependency
+  private static final Object EXTERNAL_TABLE_PROVIDER = initializeExternalTableProvider();
+  private static final java.lang.reflect.Method CREATE_TABLE_METHOD = getCreateTableMethod();
+
   protected final int formatVersion;
 
   @SuppressWarnings("checkstyle:MemberName")
@@ -199,26 +204,50 @@ public void cleanupTables() {
     TestTables.clearTables();
   }
 
+  @AfterClass
+  public static void shutdownExternalTableProvider() {
+    if (EXTERNAL_TABLE_PROVIDER != null) {
+      try {
+        java.lang.reflect.Method afterAll =
+            EXTERNAL_TABLE_PROVIDER.getClass().getMethod("afterAll");
+        afterAll.invoke(EXTERNAL_TABLE_PROVIDER);
+      } catch (Exception e) {
+        throw new RuntimeException(
+            "Failed to shutdown external table provider: "
+                + EXTERNAL_TABLE_PROVIDER.getClass().getName(),
+            e);
+      }
+    }
+  }
+
   List<File> listManifestFiles() {
     return listManifestFiles(tableDir);
   }
 
   List<File> listManifestFiles(File tableDirToList) {
-    return Lists.newArrayList(
+    File[] files =
         new File(tableDirToList, "metadata")
             .listFiles(
                 (dir, name) ->
                     !name.startsWith("snap")
-                        && Files.getFileExtension(name).equalsIgnoreCase("avro")));
+                        && Files.getFileExtension(name).equalsIgnoreCase("avro"));
+    if (files == null) {
+      return Lists.newArrayList();
+    }
+    return Lists.newArrayList(files);
   }
 
   List<File> listManifestLists(String tableDirToList) {
-    return Lists.newArrayList(
+    File[] files =
         new File(tableDirToList, "metadata")
             .listFiles(
                 (dir, name) ->
                     name.startsWith("snap")
-                        && Files.getFileExtension(name).equalsIgnoreCase("avro")));
+                        && Files.getFileExtension(name).equalsIgnoreCase("avro"));
+    if (files == null) {
+      return Lists.newArrayList();
+    }
+    return Lists.newArrayList(files);
   }
 
   public static long countAllMetadataFiles(File tableDir) {
@@ -228,19 +257,60 @@ public static long countAllMetadataFiles(File tableDir) {
   }
 
   protected TestTables.TestTable create(Schema schema, PartitionSpec spec) {
+    if (EXTERNAL_TABLE_PROVIDER != null && CREATE_TABLE_METHOD != null) {
+      try {
+        Object result =
+            CREATE_TABLE_METHOD.invoke(
+                EXTERNAL_TABLE_PROVIDER, tableDir, "test", schema, spec, formatVersion);
+        return (TestTables.TestTable) result;
+      } catch (Exception e) {
+        throw new RuntimeException("Failed to create table via external provider", e);
+      }
+    }
     return TestTables.create(tableDir, "test", schema, spec, formatVersion);
   }
 
+  private static Object initializeExternalTableProvider() {
+    String providerClass = System.getProperty("iceberg.test.table.provider");
+    if (providerClass == null || providerClass.isEmpty()) {
+      return null;
+    }
+
+    try {
+      Object provider = Class.forName(providerClass).getDeclaredConstructor().newInstance();
+      // Call beforeAll() via reflection
+      java.lang.reflect.Method beforeAll = provider.getClass().getMethod("beforeAll");
+      beforeAll.invoke(provider);
+      return provider;
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to initialize " + providerClass, e);
+    }
+  }
+
+  private static java.lang.reflect.Method getCreateTableMethod() {
+    if (EXTERNAL_TABLE_PROVIDER == null) {
+      return null;
+    }
+    try {
+      return EXTERNAL_TABLE_PROVIDER
+          .getClass()
+          .getMethod("createTable", File.class, String.class, Schema.class, PartitionSpec.class, int.class);
+    } catch (NoSuchMethodException e) {
+      throw new RuntimeException(
+          "External table provider must have createTable(File, String, Schema, PartitionSpec, int) method", e);
+    }
+  }
+
   TestTables.TestTable load() {
-    return TestTables.load(tableDir, "test");
+    return TestTables.load(tableDir, table.name());
   }
 
   Integer version() {
-    return TestTables.metadataVersion("test");
+    return TestTables.metadataVersion(table.name());
   }
 
   public TableMetadata readMetadata() {
-    return TestTables.readMetadata("test");
+    return TestTables.readMetadata(table.name());
   }
 
   ManifestFile writeManifest(DataFile... files) throws IOException {