From 49528d0d347f8cb0416455da7da37a901920e77d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:02:57 -0400 Subject: [PATCH 1/4] Onboarding calcite opensearch revision publishing setups Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 81 ++++++++++ gradle/libs.versions.toml | 5 + ...r-TCCL-for-Janino-parent-classloader.patch | 151 ++++++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 .github/workflows/calcite-snapshots.yml create mode 100644 sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml new file mode 100644 index 0000000000000..26177da2c81d4 --- /dev/null +++ b/.github/workflows/calcite-snapshots.yml @@ -0,0 +1,81 @@ +# This workflow will check out, build, and publish snapshots of lucene. + +name: OpenSearch Lucene snapshots + +on: + workflow_dispatch: + # Inputs the workflow accepts. + inputs: + ref: + description: 'Calcite ref in github.com/apache/calcite' + type: string + required: false + default: 'main' + java_version: + description: 'Java version to use' + type: string + required: false + default: '21' + +jobs: + publish-snapshots: + if: github.repository == 'opensearch-project/OpenSearch' + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + + steps: + - name: Checkout Calcite ref:${{ github.event.inputs.ref }} + uses: actions/checkout@v6 + with: + repository: 'apache/calcite' + ref: ${{ github.event.inputs.ref }} + persist-credentials: false + + - name: Checkout OpenSearch main + uses: actions/checkout@v6 + with: + repository: 'opensearch-project/OpenSearch' + ref: 'main' + persist-credentials: false + path: 'os_main' + + - name: Setup JDK ${{ github.event.inputs.java_version }} + uses: actions/setup-java@v5 + with: + java-version: ${{ github.event.inputs.java_version }} + distribution: 'temurin' + + - name: Apply Patches and build calcite jars + run: | + git apply os_main/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch + BASE_VER=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"` + REV=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite_os_rev" | grep -Eo "[0-9]+"` + CALCITE_VER=$BASE_VER-opensearch-$REV + sed -i "s/calcite\.version.*/calcite.version=$CALCITE_VER/" gradle.properties + ./gradlew :core:publishToMavenLocal :linq4j:publishToMavenLocal -Prelease -PskipSign -PskipJavadoc -x test --no-daemon + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }} + aws-region: us-east-1 + + - name: Get S3 Bucket + id: get_s3_bucket + run: | + lucene_snapshots_bucket=`aws secretsmanager get-secret-value --secret-id jenkins-artifact-bucket-name --query SecretString --output text` + echo "::add-mask::$lucene_snapshots_bucket" + echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }} + aws-region: us-east-1 + + - name: Copy files to S3 with the aws CLI + run: | + aws s3 cp ~/.m2/repository/org/apache/calcite/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/maven/org/apache/calcite/ --recursive --no-progress diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1767c3567e958..f8fc6004330df 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -104,6 +104,11 @@ opentelemetrysemconv = "1.40.0" arrow = "18.1.0" flatbuffers = "2.0.0" +# calcite is locally patched and published to OpenSearch maven snapshots; see .github/workflows/calcite-snapshots.yml. +# Published as org.apache.calcite:calcite-core:${calcite}-opensearch-${calcite_os_rev}. +calcite = "1.41.0" +calcite_os_rev = "1" + [libraries] antlr4-runtime = { group = "org.antlr", name = "antlr4-runtime", version.ref = "antlr4" } asm-analysis = { group = "org.ow2.asm", name = "asm-analysis", version.ref = "asm" } diff --git a/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch b/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch new file mode 100644 index 0000000000000..6c378de6c1686 --- /dev/null +++ b/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch @@ -0,0 +1,151 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Mustang +Date: Wed, 6 May 2026 00:00:00 -0700 +Subject: [PATCH] CALCITE-3745: TCCL-chained classloader for Janino parent CL + +Introduce a TcclChainedClassLoader utility that resolves classes via the +thread context classloader first, falling back to the Calcite-local CL +if a name is not found on TCCL. Every site that configures Janino's +parent classloader (EnumerableInterpretable, JaninoRexCompiler, +RexExecutable, JaninoRelMetadataProvider) now uses the chained loader. + +This keeps Calcite's internal types always resolvable while making +child-plugin UDFs visible when the host (OpenSearch's extendedPlugins) +sets TCCL to the child classloader. +--- + .../enumerable/EnumerableInterpretable.java | 3 +- + .../interpreter/JaninoRexCompiler.java | 3 +- + .../metadata/JaninoRelMetadataProvider.java | 4 +- + .../org/apache/calcite/rex/RexExecutable.java | 4 +- + .../calcite/util/TcclChainedClassLoader.java | 61 +++++++++++++++++++ + 5 files changed, 71 insertions(+), 4 deletions(-) + create mode 100644 core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java + +diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +index 5f32ab1..1c9ce19 100644 +--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java ++++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java +@@ -145,7 +145,8 @@ static Bindable getBindable(ClassDeclaration expr, String classBody, int fieldCo + "Unable to instantiate java compiler", e); + } + final ISimpleCompiler compiler = compilerFactory.newSimpleCompiler(); +- compiler.setParentClassLoader(classLoader); ++ compiler.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain(classLoader)); + final String s = "public final class " + expr.name + " implements " + + (fieldCount == 1 + ? Bindable.class.getCanonicalName() + ", " + Typed.class.getCanonicalName() +diff --git a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java +index bca4f85..d6de426 100644 +--- a/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java ++++ b/core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java +@@ -211,7 +211,8 @@ static Scalar.Producer getScalar(ClassDeclaration expr, String s) + IClassBodyEvaluator cbe = compilerFactory.newClassBodyEvaluator(); + cbe.setClassName(expr.name); + cbe.setImplementedInterfaces(new Class[] {Scalar.Producer.class}); +- cbe.setParentClassLoader(classLoader); ++ cbe.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain(classLoader)); + if (CalciteSystemProperty.DEBUG.value()) { + // Add line numbers to the generated janino class + cbe.setDebuggingInformation(true, true, true); +diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +index 135b11e..34a5e4b 100644 +--- a/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java ++++ b/core/src/main/java/org/apache/calcite/rel/metadata/JaninoRelMetadataProvider.java +@@ -157,7 +157,9 @@ static > MH compile(String className, + } + + final ISimpleCompiler compiler = compilerFactory.newSimpleCompiler(); +- compiler.setParentClassLoader(JaninoRexCompiler.class.getClassLoader()); ++ compiler.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain( ++ JaninoRexCompiler.class.getClassLoader())); + + if (CalciteSystemProperty.DEBUG.value()) { + // Add line numbers to the generated janino class +diff --git a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java +index 8828654..1e91951 100644 +--- a/core/src/main/java/org/apache/calcite/rex/RexExecutable.java ++++ b/core/src/main/java/org/apache/calcite/rex/RexExecutable.java +@@ -60,7 +60,9 @@ public RexExecutable(String code, Object reason) { + cbe.setClassName(GENERATED_CLASS_NAME); + cbe.setExtendedClass(Utilities.class); + cbe.setImplementedInterfaces(new Class[] {Function1.class, Serializable.class}); +- cbe.setParentClassLoader(RexExecutable.class.getClassLoader()); ++ cbe.setParentClassLoader( ++ org.apache.calcite.util.TcclChainedClassLoader.chain( ++ RexExecutable.class.getClassLoader())); + cbe.cook(new Scanner(null, new StringReader(code))); + Class c = cbe.getClazz(); + //noinspection unchecked +diff --git a/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java b/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java +new file mode 100644 +index 0000000..259d71c +--- /dev/null ++++ b/core/src/main/java/org/apache/calcite/util/TcclChainedClassLoader.java +@@ -0,0 +1,61 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to you under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.calcite.util; ++ ++/** ++ * CALCITE-3745 (OpenSearch patch): helper to build a classloader that ++ * prefers the thread context classloader for name resolution but falls back ++ * to a supplied Calcite-local classloader for Calcite's own internal types. ++ * ++ *

When Calcite is embedded under a parent plugin classloader (e.g. in ++ * OpenSearch's {@code extendedPlugins} layout), child plugins register UDFs ++ * that end up referenced by name in Janino-generated code. The default ++ * {@code SomeCalciteClass.class.getClassLoader()} cannot see those UDFs. ++ * Using TCCL alone breaks in contexts where TCCL is a stripped-down ++ * classloader that has no view of Calcite's own internal types. Chaining ++ * solves both cases. ++ */ ++public final class TcclChainedClassLoader { ++ private TcclChainedClassLoader() {} ++ ++ /** ++ * Returns a classloader that resolves classes by consulting the thread ++ * context classloader first, then falling back to {@code fallback}. If ++ * TCCL is unset or identical to {@code fallback}, the fallback is ++ * returned unchanged. ++ */ ++ public static ClassLoader chain(ClassLoader fallback) { ++ final ClassLoader tccl = Thread.currentThread().getContextClassLoader(); ++ if (tccl == null || tccl == fallback) { ++ return fallback; ++ } ++ return new ClassLoader(fallback) { ++ @Override protected Class loadClass(String name, boolean resolve) ++ throws ClassNotFoundException { ++ try { ++ Class c = tccl.loadClass(name); ++ if (resolve) { ++ resolveClass(c); ++ } ++ return c; ++ } catch (ClassNotFoundException e) { ++ return super.loadClass(name, resolve); ++ } ++ } ++ }; ++ } ++} +-- +2.50.1 (Apple Git-155) + From 03581c7c54be09e61ddebecc9f7f83b9d45a1db5 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:09:10 -0400 Subject: [PATCH 2/4] Update a comment Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml index 26177da2c81d4..2c1f3840fec5f 100644 --- a/.github/workflows/calcite-snapshots.yml +++ b/.github/workflows/calcite-snapshots.yml @@ -1,4 +1,4 @@ -# This workflow will check out, build, and publish snapshots of lucene. +# This workflow will check out, build, and publish snapshots of calcite. name: OpenSearch Lucene snapshots From 16c1353f68de50411ec153895f62cd11bd6f9f3f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:11:00 -0400 Subject: [PATCH 3/4] Update default ref to use commit ids Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml index 2c1f3840fec5f..f3abb5bb7fe46 100644 --- a/.github/workflows/calcite-snapshots.yml +++ b/.github/workflows/calcite-snapshots.yml @@ -7,10 +7,10 @@ on: # Inputs the workflow accepts. inputs: ref: - description: 'Calcite ref in github.com/apache/calcite' + description: 'Calcite ref in github.com/apache/calcite, default to calcite-1.41.0 tag (c838dd471ca36f5648ef13e5c3c34c6ca0815322)' type: string required: false - default: 'main' + default: 'c838dd471ca36f5648ef13e5c3c34c6ca0815322' java_version: description: 'Java version to use' type: string From e10c599d03f44bb4ff7c8c6e786b91b19d32d051 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 7 May 2026 17:17:33 -0400 Subject: [PATCH 4/4] Make patch file more dynamic now Signed-off-by: Peter Zhu --- .github/workflows/calcite-snapshots.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/calcite-snapshots.yml b/.github/workflows/calcite-snapshots.yml index f3abb5bb7fe46..5b1dbd971108d 100644 --- a/.github/workflows/calcite-snapshots.yml +++ b/.github/workflows/calcite-snapshots.yml @@ -16,6 +16,11 @@ on: type: string required: false default: '21' + patch_file_path: + description: 'The patch file, default to sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch' + type: string + required: false + default: 'sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch' jobs: publish-snapshots: @@ -50,7 +55,7 @@ jobs: - name: Apply Patches and build calcite jars run: | - git apply os_main/sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch + git apply os_main/${{ github.event.inputs.patch_file_path }} BASE_VER=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"` REV=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite_os_rev" | grep -Eo "[0-9]+"` CALCITE_VER=$BASE_VER-opensearch-$REV