ESQL: Add support for ORC file format

docs/changelog/142900.yaml

@@ -0,0 +1,5 @@
+area: ES|QL
+issues: []
+pr: 142900
+summary: Add support for ORC file format
+type: feature

gradle/verification-metadata.xml

@@ -97,6 +97,16 @@
             <sha256 value="ba6f338e8c02f56a7296bf669f4748f6a0506b6074452a59690da204d8f535e0" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.aayushatharva.brotli4j" name="brotli4j" version="1.18.0">
+         <artifact name="brotli4j-1.18.0.jar">
+            <sha256 value="0cab5c97314362a1d0d59206eb79ace72363ceac029f5e932acec6ce6bb23db8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.aayushatharva.brotli4j" name="service" version="1.18.0">
+         <artifact name="service-1.18.0.jar">
+            <sha256 value="6b27c85555992438530a88ae8192b15589fdc21ea22dc3c597393bf8f2dd8805" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.avast.gradle" name="gradle-docker-compose-plugin" version="0.17.5">
          <artifact name="gradle-docker-compose-plugin-0.17.5.jar">
             <sha256 value="bc818ee3015f7cea73d5a603fc7a542ad82ebb5799e406d9abc81ac42caa90f0" origin="Generated by Gradle"/>
@@ -577,6 +587,11 @@
             <sha256 value="a38a4a97f1b43b878c91e0dd7f1e3d17f0e70beec71d95bacaf56a2f207624a3" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.github.luben" name="zstd-jni" version="1.5.7-6">
+         <artifact name="zstd-jni-1.5.7-6.jar">
+            <sha256 value="8d6feb1da335f3ab13c584c613e23c7b3c61b392e37956872057baf8f0ca1d6f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.github.spotbugs" name="spotbugs-annotations" version="4.0.2">
          <artifact name="spotbugs-annotations-4.0.2.jar">
             <sha256 value="3ef6c9f822b601aa151e10e123b49e5604243a4a99bcc47e4e1f9eea9781dc63" origin="Generated by Gradle"/>
@@ -792,16 +807,16 @@
             <sha256 value="d1f3c66aa91ac52549e00ae3b208ba4b9af7d72d68f230643553beb38e6118ac" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.errorprone" name="error_prone_annotations" version="2.30.0">
-         <artifact name="error_prone_annotations-2.30.0.jar">
-            <sha256 value="144f3aefbd6e27daec55d3753b2c6b13c1afdaf0cf04816cdb564588ed92f1bd" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.3.4">
          <artifact name="error_prone_annotations-2.3.4.jar">
             <sha256 value="baf7d6ea97ce606c53e11b6854ba5f2ce7ef5c24dddf0afa18d1260bd25b002c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.errorprone" name="error_prone_annotations" version="2.30.0">
+         <artifact name="error_prone_annotations-2.30.0.jar">
+            <sha256 value="144f3aefbd6e27daec55d3753b2c6b13c1afdaf0cf04816cdb564588ed92f1bd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.36.0">
          <artifact name="error_prone_annotations-2.36.0.jar">
             <sha256 value="77440e270b0bc9a249903c5a076c36a722c4886ca4f42675f2903a1c53ed61a5" origin="Generated by Gradle"/>
@@ -1022,6 +1037,11 @@
             <sha256 value="8540247fad9e06baefa8fb45eb313802d019f485f14300e0f9d6b556ed88e753" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="3.25.8">
+         <artifact name="protobuf-java-3.25.8.jar">
+            <sha256 value="72bdb32eb38cafb7dcd288262c29a34d57cba2e19101af9685155ba8c0a56008" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java" version="4.30.2">
          <artifact name="protobuf-java-4.30.2.jar">
             <sha256 value="0f3a4e9264db07cec429f2a68a66030e9b7487277b76863cdd0e9238cece249b" origin="Generated by Gradle"/>
@@ -3285,6 +3305,11 @@
             <sha256 value="4b4f70f14576cabec34607a035b58aaefa6c042d693328d90de79db581966f8b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.hive" name="hive-storage-api" version="2.8.1">
+         <artifact name="hive-storage-api-2.8.1.jar">
+            <sha256 value="2bb77d246a9724a371301049239c6e53039efe4136b0ed34ee5a9c7b3cd861d8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.htrace" name="htrace-core4" version="4.0.1-incubating">
          <artifact name="htrace-core4-4.0.1-incubating.jar">
             <sha256 value="0abe211fbe122dc18be76fe58fc366052ec1444e7afcbb29cc1bed828710e6de" origin="Generated by Gradle"/>
@@ -3940,6 +3965,21 @@
             <sha256 value="39b2dfc8e84380bf7adab657d3d5e1625cb6592a885ebdb854ec5c6f7a3ec88d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.orc" name="orc-core" version="2.2.2">
+         <artifact name="orc-core-2.2.2.jar">
+            <sha256 value="fac6d35cabb136f62932b17a687997ff5dd7e139ce1af4975f602fb8c7f0598a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.orc" name="orc-format" version="1.1.1">
+         <artifact name="orc-format-1.1.1.jar">
+            <sha256 value="41d4e038913ba2ca5e7b9bd5a90c6f641879ef519b9763f39d7eb45e6e64e415" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.orc" name="orc-shims" version="2.2.2">
+         <artifact name="orc-shims-2.2.2.jar">
+            <sha256 value="46c730ca9aef90bd9df03bdfc50564a51a4cf5da9f2888e97b9b0faa1e88c260" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.parquet" name="parquet-avro" version="1.16.0">
          <artifact name="parquet-avro-1.16.0.jar">
             <sha256 value="31748bb5006d2f86ee1853754ea9d4bae26784d1a651aecd6fb87faf770ca0ba" origin="Generated by Gradle"/>
@@ -5563,6 +5603,11 @@
             <sha256 value="18c4a0095d5c1da6b817592e767bb23d29dd2f560ad74df75ff3961dbde25b79" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-api" version="1.7.30">
+         <artifact name="slf4j-api-1.7.30.jar">
+            <sha256 value="cdba07964d1bb40a0761485c6b1e8c2f8fd9eb1d19c53928ac0d7f9510105c57" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-api" version="1.7.32">
          <artifact name="slf4j-api-1.7.32.jar">
             <sha256 value="3624f8474c1af46d75f98bc097d7864a323c81b3808aa43689a6e1c601c027be" origin="Generated by Gradle"/>
@@ -5653,6 +5698,11 @@
             <sha256 value="edd5740fd67f1a1edd7dae4b08eef333c81cbe84a9d437f14ecdb9b82bc04688" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.threeten" name="threeten-extra" version="1.8.0">
+         <artifact name="threeten-extra-1.8.0.jar">
+            <sha256 value="51e4d21edc4e9447f7760c050e0baee75d7d973f387ba605a17abdc4d24fd6d8" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.threeten" name="threetenbp" version="1.7.0">
          <artifact name="threetenbp-1.7.0.jar">
             <sha256 value="857917d2319a4e92dc1c5e3aeb75a0dac84445ed315e7ac3d82bb8d2b298977f" origin="Generated by Gradle"/>

x-pack/plugin/esql-datasource-orc/build.gradle

@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.publish'
+
+versions << [
+  'hadoop' : '3.4.2',
+  'orc'    : '2.2.2',
+]
+
+esplugin {
+  name = 'esql-datasource-orc'
+  description = 'ORC format support for ESQL external data sources'
+  classname = 'org.elasticsearch.xpack.esql.datasource.orc.OrcDataSourcePlugin'
+  extendedPlugins = ['x-pack-esql']
+}
+
+base {
+  archivesName = 'esql-datasource-orc'
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('esql'))
+  compileOnly project(path: xpackModule('esql-core'))
+  compileOnly project(path: xpackModule('core'))
+  compileOnly project(':server')
+  compileOnly project(xpackModule('esql:compute'))
+
+  implementation("org.apache.orc:orc-core:${versions.orc}") {
+    exclude group: 'org.slf4j', module: 'slf4j-api'
+  }
+
+  // Hive storage API - provides VectorizedRowBatch and ColumnVector classes used by ORC's
+  // reader API. ORC shades this during its Maven build but the published jar still references
+  // these classes in its public API (TypeDescription.createRowBatch(), Reader.rows(), etc.)
+  implementation('org.apache.hive:hive-storage-api:2.8.1')
+
+  // Protobuf - required by orc-format's generated classes (OrcProto) which reference
+  // com.google.protobuf.MessageOrBuilder at runtime
+  implementation('com.google.protobuf:protobuf-java:3.25.8')
+
+  // Hadoop dependencies - required because ORC's Reader API uses Hadoop Configuration,
+  // Path, and FileSystem in its public interfaces.
+  implementation('org.apache.hadoop:hadoop-client-api:3.4.2') {
+    exclude group: 'org.slf4j', module: 'slf4j-api'
+  }
+  implementation('org.apache.hadoop:hadoop-client-runtime:3.4.2') {
+    exclude group: 'org.slf4j', module: 'slf4j-api'
+  }
+
+  // Re-add slf4j-api at the platform version to avoid jar hell with x-pack-core
+  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+
+  testImplementation project(':test:framework')
+  testImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /lucene-.*/, to: 'lucene'
+  mapping from: /orc-.*/, to: 'orc'
+  mapping from: /hadoop-.*/, to: 'hadoop'
+  mapping from: /hive-.*/, to: 'orc'
+  mapping from: /aircompressor-.*/, to: 'orc'
+  mapping from: /threeten-extra-.*/, to: 'orc'
+  mapping from: /protobuf-.*/, to: 'orc'
+  mapping from: /brotli4j-.*/, to: 'orc'
+  mapping from: /service-.*/, to: 'orc'
+  mapping from: /zstd-jni-.*/, to: 'orc'
+  mapping from: /jts-core-.*/, to: 'orc'
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses()
+  ignoreViolations(
+    // Hadoop internal uses sun.misc.Unsafe
+    'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
+    'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',
+    'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
+    'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
+    'org.apache.hadoop.io.nativeio.NativeIO',
+    'org.apache.hadoop.service.launcher.InterruptEscalator',
+    'org.apache.hadoop.service.launcher.IrqHandler',
+    'org.apache.hadoop.util.SignalLogger$Handler',
+    // Hadoop shaded Guava uses sun.misc.Unsafe
+    'org.apache.hadoop.shaded.com.google.common.cache.Striped64',
+    'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1',
+    'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell',
+    'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+    'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+    'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+    'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+    'org.apache.hadoop.shaded.com.google.common.hash.Striped64',
+    'org.apache.hadoop.shaded.com.google.common.hash.Striped64$1',
+    'org.apache.hadoop.shaded.com.google.common.hash.Striped64$Cell',
+    'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+    // Hadoop shaded Avro uses sun.misc.Unsafe
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField',
+    'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
+    // Hadoop shaded Curator Guava uses sun.misc.Unsafe
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$1',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$Cell',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$1',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$Cell',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+    'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor',
+    // Hadoop thirdparty Protobuf uses sun.misc.Unsafe
+    'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
+    // Hadoop thirdparty Guava uses sun.misc.Unsafe
+    'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64',
+    'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$1',
+    'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$Cell',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
+    'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
+    'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+    // Protobuf uses sun.misc.Unsafe
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
+  )
+}