Sandbox/qa - Add multi shard qa test suite

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AggregationsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Aggregation functions testing PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return AggregationsTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testAggregationsPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: distinct_count/percentile value mismatches (approx + HLL merge). Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(7, 8, 9, 10);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppLogsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Application log analysis PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return AppLogsTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testAppLogsPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: unsupported operations. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(5, 9);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/AppendPipeCommandIT.java

@@ -56,7 +56,6 @@ protected void onBeforeQuery() throws IOException {
 
     // ── duplicate + inline sort, then head ──────────────────────────────────────
 
-    @org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/21626")
     public void testAppendPipeSort() throws IOException {
         // Branch: stats sum(int0) by str0 → 3 rows (FURNITURE=1, OFFICE SUPPLIES=18, TECHNOLOGY=49).
         // `appendpipe [sort -sum_int0_by_str0]` duplicates them desc-sorted and appends. `head 5`

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ComplexJoinsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Complex Joins PPL integration test (multi-index). Tests join operations across multiple indexes.
@@ -34,9 +33,14 @@ private void ensureAdditionalDataProvisioned() throws Exception {
         }
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testComplexJoinsPplQueries() throws Exception {
         ensureAdditionalDataProvisioned();
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: join row-count / unsupported shapes. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(1, 2, 3, 4, 7, 8, 9, 10);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ExtensiveCoveragePplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Extensive function coverage testing PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return ExtensiveCoverageTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testExtensiveCoveragePplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: mixed: date/time formatting, string-value, unsupported-fn (see per-q). Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(8, 9, 10, 13, 19, 20, 22, 24, 25, 28, 29, 30, 37, 39, 40, 41, 42, 43, 44, 52, 54, 55, 56, 57, 58, 59, 60, 61, 62, 70, 77, 81, 85, 86, 88, 89, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 108, 110, 111, 112, 114, 115, 116, 117, 119, 120, 125, 126, 128, 129, 130, 131, 132, 136, 137, 138, 139, 143, 144, 145, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 160, 162, 163, 177, 188, 189, 190, 191, 193, 195, 196);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FulltextWindowPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Full-text search with window functions testing PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return FulltextWindowTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testFulltextWindowPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: fulltext + window combinations unsupported. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(1, 6, 8, 12, 13, 14, 15, 17, 19);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/FunctionsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * PPL function testing PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return FunctionsTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testFunctionsPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: split() unsupported. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(13);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/IpFieldMultiShardIT.java

@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.qa;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Multi-shard {@code ip}-field coverage (Mustang bug tracker Bug 3): a query over an index with an
+ * {@code ip}-type field can fail with a Substrait {@code Binary} vs {@code BinaryView} schema mismatch.
+ * {@link FieldTypeCoverageIT#testIp()} covers {@code ip} at 1 shard; this exercises the 2-shard reduce
+ * path. Run unmuted — if the defect is present it fails here, otherwise it passes.
+ */
+public class IpFieldMultiShardIT extends AnalyticsRestTestCase {
+
+    private static final Dataset DATASET = new Dataset("ip_multishard", "ip_multishard");
+
+    private static boolean provisioned = false;
+
+    @Override
+    protected void onBeforeQuery() throws IOException {
+        if (provisioned == false) {
+            DatasetProvisioner.provision(client(), DATASET, 2);
+            provisioned = true;
+        }
+    }
+
+    /** count() by status_code over the 10 docs (status_code = 200 + id%3): 200->4, 201->3, 202->3. */
+    @SuppressWarnings("unchecked")
+    public void testAggregationOnIndexWithIpFieldAtTwoShards() throws IOException {
+        Map<String, Object> resp = executePpl("source=" + DATASET.indexName + " | stats count() by status_code");
+        List<List<Object>> rows = (List<List<Object>>) resp.get("datarows");
+        assertNotNull("expected datarows for count-by-status over an ip-bearing index", rows);
+
+        Map<Integer, Integer> counts = new HashMap<>();
+        for (List<Object> row : rows) {
+            counts.put(((Number) row.get(1)).intValue(), ((Number) row.get(0)).intValue());
+        }
+        Map<Integer, Integer> expected = Map.of(200, 4, 201, 3, 202, 3);
+        assertEquals("count() by status_code", expected, counts);
+    }
+}

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/KubernetesLogsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Kubernetes log analysis PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return KubernetesLogsTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testKubernetesLogsPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: unsupported operation. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(9);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MatchLikeParityIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 
@@ -58,7 +57,6 @@
  * {@code OpenSearchTestCase}, so the seed is printed on failure. Re-run with
  * {@code ./gradlew :...:integTest -Dtests.seed=HEX} to reproduce.
  */
-@LuceneTestCase.AwaitsFix(bugUrl = "")
 public class MatchLikeParityIT extends AnalyticsRestTestCase {
 
     private static final String INDEX_NAME = "match_like_parity";

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiIndexQueriesPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Multi-Index Queries PPL integration test (multi-index). Tests fields, rename, top, rare, span commands.
@@ -33,9 +32,14 @@ private void ensureAdditionalDataProvisioned() throws Exception {
         }
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testMultiIndexQueriesPplQueries() throws Exception {
         ensureAdditionalDataProvisioned();
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: multi-index 'one concrete index' limit. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(2, 7, 10);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MultiSourceJoinsPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Complex Redesigned (multi-index) PPL integration test. Runs PPL queries against complex_redesigned data.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return MultiSourceJoinsTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testMultiSourceJoinsPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: multi-source join unsupported. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(2, 4);
+    }
 }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/ResponseValidator.java

@@ -93,8 +93,22 @@ private static Map<String, Object> parseSimpleJson(String json) throws IOExcepti
      * Compare expected and actual responses, focusing on data rows.
      * Rows are compared in an unordered fashion - both sets are sorted before comparison.
      */
-    @SuppressWarnings("unchecked")
     private static String compareResponses(Map<String, Object> expected, Map<String, Object> actual, String language, int queryNumber) {
+        String label = language.toUpperCase(java.util.Locale.ROOT) + " Q" + queryNumber;
+        return compareData(expected, actual, label);
+    }
+
+    /**
+     * Compare two response maps by their data rows, unordered and numeric-tolerant.
+     * Returns {@code null} when equal, otherwise a human-readable diff prefixed with
+     * {@code label}.
+     *
+     * <p>Exposed for name-keyed suites (e.g. the 2-shard reduce suite) that don't use the
+     * numeric {@code q{N}} scheme and for differential checks (comparing a 1-shard result
+     * against a 2-shard result rather than against a golden file).
+     */
+    @SuppressWarnings("unchecked")
+    public static String compareData(Map<String, Object> expected, Map<String, Object> actual, String label) {
         // Extract datarows from both responses
         List<List<Object>> expectedRows = extractDataRows(expected);
         List<List<Object>> actualRows = extractDataRows(actual);
@@ -104,21 +118,21 @@ private static String compareResponses(Map<String, Object> expected, Map<String,
         }
 
         if (expectedRows == null) {
-            return String.format(java.util.Locale.ROOT, "%s Q%d: Expected empty response but got %d rows",
-                language.toUpperCase(java.util.Locale.ROOT), queryNumber, actualRows.size());
+            return String.format(java.util.Locale.ROOT, "%s: Expected empty response but got %d rows", label, actualRows.size());
         }
 
         if (actualRows == null) {
-            return String.format(java.util.Locale.ROOT, "%s Q%d: Expected %d rows but got empty response",
-                language.toUpperCase(java.util.Locale.ROOT), queryNumber, expectedRows.size());
+            return String.format(java.util.Locale.ROOT, "%s: Expected %d rows but got empty response", label, expectedRows.size());
         }
 
         if (expectedRows.size() != actualRows.size()) {
-            return String.format(java.util.Locale.ROOT, "%s Q%d: Row count mismatch - expected %d, got %d",
-                language.toUpperCase(java.util.Locale.ROOT), queryNumber, expectedRows.size(), actualRows.size());
+            return String.format(java.util.Locale.ROOT, "%s: Row count mismatch - expected %d, got %d",
+                label, expectedRows.size(), actualRows.size());
         }
 
-        // Sort both row sets for unordered comparison
+        // Copy before sorting so we never reorder the caller's lists.
+        expectedRows = new java.util.ArrayList<>(expectedRows);
+        actualRows = new java.util.ArrayList<>(actualRows);
         expectedRows.sort(new RowComparator());
         actualRows.sort(new RowComparator());
 
@@ -128,14 +142,14 @@ private static String compareResponses(Map<String, Object> expected, Map<String,
             List<Object> actualRow = actualRows.get(i);
 
             if (expectedRow.size() != actualRow.size()) {
-                return String.format(java.util.Locale.ROOT, "%s Q%d row %d: Column count mismatch - expected %d, got %d",
-                    language.toUpperCase(java.util.Locale.ROOT), queryNumber, i, expectedRow.size(), actualRow.size());
+                return String.format(java.util.Locale.ROOT, "%s row %d: Column count mismatch - expected %d, got %d",
+                    label, i, expectedRow.size(), actualRow.size());
             }
 
             for (int j = 0; j < expectedRow.size(); j++) {
                 if (!valuesEqual(expectedRow.get(j), actualRow.get(j))) {
-                    return String.format(java.util.Locale.ROOT, "%s Q%d row %d col %d: Value mismatch - expected %s, got %s",
-                        language.toUpperCase(java.util.Locale.ROOT), queryNumber, i, j, expectedRow.get(j), actualRow.get(j));
+                    return String.format(java.util.Locale.ROOT, "%s row %d col %d: Value mismatch - expected %s, got %s",
+                        label, i, j, expectedRow.get(j), actualRow.get(j));
                 }
             }
         }

sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/RexCommandPplIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.analytics.qa;
 
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 
 /**
  * Rex command testing PPL integration test.
@@ -20,8 +19,13 @@ protected Dataset getDataset() {
         return RexCommandTestHelper.DATASET;
     }
 
-    @AwaitsFix(bugUrl = "Failing due to unsupported operations")
     public void testRexCommandPplQueries() throws Exception {
         runPplQueries();
     }
+
+    /** Queries that fail at 1 shard: rex unsupported / value mismatch. Skipped so the rest run and are visible. */
+    @Override
+    protected java.util.Set<Integer> getSkipQueries() {
+        return java.util.Set.of(1, 5, 7, 8, 13, 18);
+    }
 }