readScriptSpec() throws Exception {
+ return readExternalSpecTests("/external-*.csv-spec");
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-csv/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.csv b/x-pack/plugin/esql-datasource-csv/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.csv
new file mode 100644
index 0000000000000..58e6efd9a380c
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-csv/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.csv
@@ -0,0 +1,101 @@
+emp_no:integer,first_name:keyword,last_name:keyword,birth_date:date,gender:keyword,hire_date:date,languages:integer,languages.long:long,height:double,height.float:double,height.scaled_float:double,height.half_float:double,salary:integer,still_hired:boolean,avg_worked_seconds:long
+10001,Georgi,Facello,1953-09-02T00:00:00.000Z,M,1986-06-26T00:00:00.000Z,2,2,2.03,2.03,2.03,2.03,57305,true,268728049
+10002,Bezalel,Simmel,1964-06-02T00:00:00.000Z,F,1985-11-21T00:00:00.000Z,5,5,2.08,2.08,2.08,2.08,56371,true,328922887
+10003,Parto,Bamford,1959-12-03T00:00:00.000Z,M,1986-08-28T00:00:00.000Z,4,4,1.83,1.83,1.83,1.83,61805,false,200296405
+10004,Chirstian,Koblick,1954-05-01T00:00:00.000Z,M,1986-12-01T00:00:00.000Z,5,5,1.78,1.78,1.78,1.78,36174,true,311267831
+10005,Kyoichi,Maliniak,1955-01-21T00:00:00.000Z,M,1989-09-12T00:00:00.000Z,1,1,2.05,2.05,2.05,2.05,63528,true,244294991
+10006,Anneke,Preusig,1953-04-20T00:00:00.000Z,F,1989-06-02T00:00:00.000Z,3,3,1.56,1.56,1.56,1.56,60335,false,372957040
+10007,Tzvetan,Zielinski,1957-05-23T00:00:00.000Z,F,1989-02-10T00:00:00.000Z,4,4,1.70,1.70,1.70,1.70,74572,true,393084805
+10008,Saniya,Kalloufi,1958-02-19T00:00:00.000Z,M,1994-09-15T00:00:00.000Z,2,2,2.10,2.10,2.10,2.10,43906,true,283074758
+10009,Sumant,Peac,1952-04-19T00:00:00.000Z,F,1985-02-18T00:00:00.000Z,1,1,1.85,1.85,1.85,1.85,66174,false,236805489
+10010,Duangkaew,Piveteau,1963-06-01T00:00:00.000Z,,1989-08-24T00:00:00.000Z,4,4,1.70,1.70,1.70,1.70,45797,false,315236372
+10011,Mary,Sluis,1953-11-07T00:00:00.000Z,,1990-01-22T00:00:00.000Z,5,5,1.50,1.50,1.50,1.50,31120,true,239615525
+10012,Patricio,Bridgland,1960-10-04T00:00:00.000Z,,1992-12-18T00:00:00.000Z,5,5,1.97,1.97,1.97,1.97,48942,false,365510850
+10013,Eberhardt,Terkki,1963-06-07T00:00:00.000Z,,1985-10-20T00:00:00.000Z,1,1,1.94,1.94,1.94,1.94,48735,true,253864340
+10014,Berni,Genin,1956-02-12T00:00:00.000Z,,1987-03-11T00:00:00.000Z,5,5,1.99,1.99,1.99,1.99,37137,false,225049139
+10015,Guoxiang,Nooteboom,1959-08-19T00:00:00.000Z,,1987-07-02T00:00:00.000Z,5,5,1.66,1.66,1.66,1.66,25324,true,390266432
+10016,Kazuhito,Cappelletti,1961-05-02T00:00:00.000Z,,1995-01-27T00:00:00.000Z,2,2,1.54,1.54,1.54,1.54,61358,false,253029411
+10017,Cristinel,Bouloucos,1958-07-06T00:00:00.000Z,,1993-08-03T00:00:00.000Z,2,2,1.74,1.74,1.74,1.74,58715,false,236703986
+10018,Kazuhide,Peha,1954-06-19T00:00:00.000Z,,1987-04-03T00:00:00.000Z,2,2,1.97,1.97,1.97,1.97,56760,false,309604079
+10019,Lillian,Haddadi,1953-01-23T00:00:00.000Z,,1999-04-30T00:00:00.000Z,1,1,2.06,2.06,2.06,2.06,73717,false,342855721
+10020,Mayuko,Warwick,1952-12-24T00:00:00.000Z,M,1991-01-26T00:00:00.000Z,,,1.41,1.41,1.41,1.41,40031,false,373309605
+10021,Ramzi,Erde,1960-02-20T00:00:00.000Z,M,1988-02-10T00:00:00.000Z,,,1.47,1.47,1.47,1.47,60408,false,287654610
+10022,Shahaf,Famili,1952-07-08T00:00:00.000Z,M,1995-08-22T00:00:00.000Z,,,1.82,1.82,1.82,1.82,48233,false,233521306
+10023,Bojan,Montemayor,1953-09-29T00:00:00.000Z,F,1989-12-17T00:00:00.000Z,,,1.75,1.75,1.75,1.75,47896,true,330870342
+10024,Suzette,Pettey,1958-09-05T00:00:00.000Z,F,1997-05-19T00:00:00.000Z,,,2.08,2.08,2.08,2.08,64675,true,367717671
+10025,Prasadram,Heyers,1958-10-31T00:00:00.000Z,M,1987-08-17T00:00:00.000Z,,,1.87,1.87,1.87,1.87,47411,false,371270797
+10026,Yongqiao,Berztiss,1953-04-03T00:00:00.000Z,M,1995-03-20T00:00:00.000Z,,,2.10,2.10,2.10,2.10,28336,true,359208133
+10027,Divier,Reistad,1962-07-10T00:00:00.000Z,F,1989-07-07T00:00:00.000Z,,,1.53,1.53,1.53,1.53,73851,false,374037782
+10028,Domenick,Tempesti,1963-11-26T00:00:00.000Z,M,1991-10-22T00:00:00.000Z,,,2.07,2.07,2.07,2.07,39356,true,226435054
+10029,Otmar,Herbst,1956-12-13T00:00:00.000Z,M,1985-11-20T00:00:00.000Z,,,1.99,1.99,1.99,1.99,74999,false,257694181
+10030,,Demeyer,1958-07-14T00:00:00.000Z,M,1994-02-17T00:00:00.000Z,3,3,1.92,1.92,1.92,1.92,67492,false,394597613
+10031,,Joslin,1959-01-27T00:00:00.000Z,M,1991-09-01T00:00:00.000Z,4,4,1.68,1.68,1.68,1.68,37716,false,348545109
+10032,,Reistad,1960-08-09T00:00:00.000Z,F,1990-06-20T00:00:00.000Z,3,3,2.10,2.10,2.10,2.10,62233,false,277622619
+10033,,Merlo,1956-11-14T00:00:00.000Z,M,1987-03-18T00:00:00.000Z,1,1,1.63,1.63,1.63,1.63,70011,false,208374744
+10034,,Swan,1962-12-29T00:00:00.000Z,M,1988-09-21T00:00:00.000Z,1,1,1.46,1.46,1.46,1.46,39878,false,214393176
+10035,,Chappelet,1953-02-08T00:00:00.000Z,M,1988-09-05T00:00:00.000Z,5,5,1.81,1.81,1.81,1.81,25945,false,203838153
+10036,,Portugali,1959-08-10T00:00:00.000Z,M,1992-01-03T00:00:00.000Z,4,4,1.61,1.61,1.61,1.61,60781,false,305493131
+10037,,Makrucki,1963-07-22T00:00:00.000Z,M,1990-12-05T00:00:00.000Z,2,2,2.00,2.00,2.00,2.00,37691,true,359217000
+10038,,Lortz,1960-07-20T00:00:00.000Z,M,1989-09-20T00:00:00.000Z,4,4,1.53,1.53,1.53,1.53,35222,true,314036411
+10039,,Brender,1959-10-01T00:00:00.000Z,M,1988-01-19T00:00:00.000Z,2,2,1.55,1.55,1.55,1.55,36051,false,243221262
+10040,Weiyi,Meriste,,F,1993-02-14T00:00:00.000Z,4,4,1.90,1.90,1.90,1.90,37112,false,244478622
+10041,Uri,Lenart,,F,1989-11-12T00:00:00.000Z,1,1,1.75,1.75,1.75,1.75,56415,false,287789442
+10042,Magy,Stamatiou,,F,1993-03-21T00:00:00.000Z,3,3,1.44,1.44,1.44,1.44,30404,true,246355863
+10043,Yishay,Tzvieli,,M,1990-10-20T00:00:00.000Z,1,1,1.52,1.52,1.52,1.52,34341,true,287222180
+10044,Mingsen,Casley,,F,1994-05-21T00:00:00.000Z,1,1,2.06,2.06,2.06,2.06,39728,false,387408356
+10045,Moss,Shanbhogue,,M,1989-09-02T00:00:00.000Z,3,3,1.70,1.70,1.70,1.70,74970,false,371418933
+10046,Lucien,Rosenbaum,,M,1992-06-20T00:00:00.000Z,4,4,1.52,1.52,1.52,1.52,50064,true,302353405
+10047,Zvonko,Nyanchama,,M,1989-03-31T00:00:00.000Z,4,4,1.52,1.52,1.52,1.52,42716,true,306369346
+10048,Florian,Syrotiuk,,M,1985-02-24T00:00:00.000Z,3,3,2.00,2.00,2.00,2.00,26436,false,248451647
+10049,Basil,Tramer,,F,1992-05-04T00:00:00.000Z,5,5,1.52,1.52,1.52,1.52,37853,true,320725709
+10050,Yinghua,Dredge,1958-05-21T00:00:00.000Z,M,1990-12-25T00:00:00.000Z,2,2,1.96,1.96,1.96,1.96,43026,true,242731798
+10051,Hidefumi,Caine,1953-07-28T00:00:00.000Z,M,1992-10-15T00:00:00.000Z,3,3,1.89,1.89,1.89,1.89,58121,true,374753122
+10052,Heping,Nitsch,1961-02-26T00:00:00.000Z,M,1988-05-21T00:00:00.000Z,1,1,1.79,1.79,1.79,1.79,55360,true,299654717
+10053,Sanjiv,Zschoche,1954-09-13T00:00:00.000Z,F,1986-02-04T00:00:00.000Z,3,3,1.58,1.58,1.58,1.58,54462,false,368103911
+10054,Mayumi,Schueller,1957-04-04T00:00:00.000Z,M,1995-03-13T00:00:00.000Z,4,4,1.82,1.82,1.82,1.82,65367,false,297441693
+10055,Georgy,Dredge,1956-06-06T00:00:00.000Z,M,1992-04-27T00:00:00.000Z,5,5,2.04,2.04,2.04,2.04,49281,false,283157844
+10056,Brendon,Bernini,1961-09-01T00:00:00.000Z,F,1990-02-01T00:00:00.000Z,2,2,1.57,1.57,1.57,1.57,33370,true,349086555
+10057,Ebbe,Callaway,1954-05-30T00:00:00.000Z,F,1992-01-15T00:00:00.000Z,4,4,1.59,1.59,1.59,1.59,27215,true,324356269
+10058,Berhard,McFarlin,1954-10-01T00:00:00.000Z,M,1987-04-13T00:00:00.000Z,3,3,1.83,1.83,1.83,1.83,38376,false,268378108
+10059,Alejandro,McAlpine,1953-09-19T00:00:00.000Z,F,1991-06-26T00:00:00.000Z,2,2,1.48,1.48,1.48,1.48,44307,false,237368465
+10060,Breannda,Billingsley,1961-10-15T00:00:00.000Z,M,1987-11-02T00:00:00.000Z,2,2,1.42,1.42,1.42,1.42,29175,true,341158890
+10061,Tse,Herber,1962-10-19T00:00:00.000Z,M,1985-09-17T00:00:00.000Z,1,1,1.45,1.45,1.45,1.45,49095,false,327550310
+10062,Anoosh,Peyn,1961-11-02T00:00:00.000Z,M,1991-08-30T00:00:00.000Z,3,3,1.70,1.70,1.70,1.70,65030,false,203989706
+10063,Gino,Leonhardt,1952-08-06T00:00:00.000Z,F,1989-04-08T00:00:00.000Z,3,3,1.78,1.78,1.78,1.78,52121,true,214068302
+10064,Udi,Jansch,1959-04-07T00:00:00.000Z,M,1985-11-20T00:00:00.000Z,5,5,1.93,1.93,1.93,1.93,33956,false,307364077
+10065,Satosi,Awdeh,1963-04-14T00:00:00.000Z,M,1988-05-18T00:00:00.000Z,2,2,1.59,1.59,1.59,1.59,50249,false,372660279
+10066,Kwee,Schusler,1952-11-13T00:00:00.000Z,M,1986-02-26T00:00:00.000Z,5,5,2.10,2.10,2.10,2.10,31897,true,360906451
+10067,Claudi,Stavenow,1953-01-07T00:00:00.000Z,M,1987-03-04T00:00:00.000Z,2,2,1.77,1.77,1.77,1.77,52044,true,347664141
+10068,Charlene,Brattka,1962-11-26T00:00:00.000Z,M,1987-08-07T00:00:00.000Z,3,3,1.58,1.58,1.58,1.58,28941,true,233999584
+10069,Margareta,Bierman,1960-09-06T00:00:00.000Z,F,1989-11-05T00:00:00.000Z,5,5,1.77,1.77,1.77,1.77,41933,true,366512352
+10070,Reuven,Garigliano,1955-08-20T00:00:00.000Z,M,1985-10-14T00:00:00.000Z,3,3,1.77,1.77,1.77,1.77,54329,true,347188604
+10071,Hisao,Lipner,1958-01-21T00:00:00.000Z,M,1987-10-01T00:00:00.000Z,2,2,2.07,2.07,2.07,2.07,40612,false,306671693
+10072,Hironoby,Sidou,1952-05-15T00:00:00.000Z,F,1988-07-21T00:00:00.000Z,5,5,1.82,1.82,1.82,1.82,54518,true,209506065
+10073,Shir,McClurg,1954-02-23T00:00:00.000Z,M,1991-12-01T00:00:00.000Z,4,4,1.66,1.66,1.66,1.66,32568,false,314930367
+10074,Mokhtar,Bernatsky,1955-08-28T00:00:00.000Z,F,1990-08-13T00:00:00.000Z,5,5,1.64,1.64,1.64,1.64,38992,true,382397583
+10075,Gao,Dolinsky,1960-03-09T00:00:00.000Z,F,1987-03-19T00:00:00.000Z,5,5,1.94,1.94,1.94,1.94,51956,false,370238919
+10076,Erez,Ritzmann,1952-06-13T00:00:00.000Z,F,1985-07-09T00:00:00.000Z,3,3,1.83,1.83,1.83,1.83,62405,false,376240317
+10077,Mona,Azuma,1964-04-18T00:00:00.000Z,M,1990-03-02T00:00:00.000Z,5,5,1.68,1.68,1.68,1.68,46595,false,351960222
+10078,Danel,Mondadori,1959-12-25T00:00:00.000Z,F,1987-05-26T00:00:00.000Z,2,2,1.81,1.81,1.81,1.81,69904,true,377116038
+10079,Kshitij,Gils,1961-10-05T00:00:00.000Z,F,1986-03-27T00:00:00.000Z,2,2,1.59,1.59,1.59,1.59,32263,false,320953330
+10080,Premal,Baek,1957-12-03T00:00:00.000Z,M,1985-11-19T00:00:00.000Z,5,5,1.80,1.80,1.80,1.80,52833,false,239266137
+10081,Zhongwei,Rosen,1960-12-17T00:00:00.000Z,M,1986-10-30T00:00:00.000Z,2,2,1.44,1.44,1.44,1.44,50128,true,321375511
+10082,Parviz,Lortz,1963-09-09T00:00:00.000Z,M,1990-01-03T00:00:00.000Z,4,4,1.61,1.61,1.61,1.61,49818,false,232522994
+10083,Vishv,Zockler,1959-07-23T00:00:00.000Z,M,1987-03-31T00:00:00.000Z,1,1,1.42,1.42,1.42,1.42,39110,false,331236443
+10084,Tuval,Kalloufi,1960-05-25T00:00:00.000Z,M,1995-12-15T00:00:00.000Z,1,1,1.51,1.51,1.51,1.51,28035,true,359067056
+10085,Kenroku,Malabarba,1962-11-07T00:00:00.000Z,M,1994-04-09T00:00:00.000Z,5,5,2.01,2.01,2.01,2.01,35742,true,353404008
+10086,Somnath,Foote,1962-11-19T00:00:00.000Z,M,1990-02-16T00:00:00.000Z,1,1,1.74,1.74,1.74,1.74,68547,true,328580163
+10087,Xinglin,Eugenio,1959-07-23T00:00:00.000Z,F,1986-09-08T00:00:00.000Z,5,5,1.74,1.74,1.74,1.74,32272,true,305782871
+10088,Jungsoon,Syrzycki,1954-02-25T00:00:00.000Z,F,1988-09-02T00:00:00.000Z,5,5,1.91,1.91,1.91,1.91,39638,false,330714423
+10089,Sudharsan,Flasterstein,1963-03-21T00:00:00.000Z,F,1986-08-12T00:00:00.000Z,4,4,1.57,1.57,1.57,1.57,43602,true,232951673
+10090,Kendra,Hofting,1961-05-30T00:00:00.000Z,M,1986-03-14T00:00:00.000Z,2,2,2.03,2.03,2.03,2.03,44956,true,212460105
+10091,Amabile,Gomatam,1955-10-04T00:00:00.000Z,M,1992-11-18T00:00:00.000Z,3,3,2.09,2.09,2.09,2.09,38645,true,242582807
+10092,Valdiodio,Niizuma,1964-10-18T00:00:00.000Z,F,1989-09-22T00:00:00.000Z,1,1,1.75,1.75,1.75,1.75,25976,false,313407352
+10093,Sailaja,Desikan,1964-06-11T00:00:00.000Z,M,1996-11-05T00:00:00.000Z,3,3,1.69,1.69,1.69,1.69,45656,false,315904921
+10094,Arumugam,Ossenbruggen,1957-05-25T00:00:00.000Z,F,1987-04-18T00:00:00.000Z,5,5,2.10,2.10,2.10,2.10,66817,false,332920135
+10095,Hilari,Morton,1965-01-03T00:00:00.000Z,M,1986-07-15T00:00:00.000Z,4,4,1.55,1.55,1.55,1.55,37702,false,321850475
+10096,Jayson,Mandell,1954-09-16T00:00:00.000Z,M,1990-01-14T00:00:00.000Z,4,4,1.94,1.94,1.94,1.94,43889,false,204381503
+10097,Remzi,Waschkowski,1952-02-27T00:00:00.000Z,M,1990-09-15T00:00:00.000Z,3,3,1.53,1.53,1.53,1.53,71165,false,206258084
+10098,Sreekrishna,Servieres,1961-09-23T00:00:00.000Z,F,1985-05-13T00:00:00.000Z,4,4,2.00,2.00,2.00,2.00,44817,false,272392146
+10099,Valter,Sullins,1956-05-25T00:00:00.000Z,F,1988-10-18T00:00:00.000Z,2,2,1.81,1.81,1.81,1.81,73578,true,377713748
+10100,Hironobu,Haraldson,1953-04-21T00:00:00.000Z,F,1987-09-21T00:00:00.000Z,4,4,1.77,1.77,1.77,1.77,68431,true,223910853
diff --git a/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvDataSourcePlugin.java b/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvDataSourcePlugin.java
new file mode 100644
index 0000000000000..8a2fcff1a14b5
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvDataSourcePlugin.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.csv;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReaderFactory;
+
+import java.util.Map;
+
+/**
+ * Data source plugin that provides CSV format support for ESQL external data sources.
+ *
+ * This plugin provides:
+ *
+ * CSV format reader for reading CSV files from any storage provider
+ *
+ *
+ * The CSV format reader uses Jackson's CSV parser for robust CSV parsing with
+ * proper quote and escape handling. It supports:
+ *
+ * Schema discovery from CSV file headers (column_name:type_name format)
+ * Column projection for efficient reads
+ * Batch reading with configurable batch sizes
+ * Direct conversion to ESQL Page format
+ *
+ *
+ * The Jackson CSV dependency is isolated in this module to keep
+ * the core ESQL plugin free of third-party format libraries.
+ */
+public class CsvDataSourcePlugin extends Plugin implements DataSourcePlugin {
+
+ @Override
+ public Map formatReaders(Settings settings) {
+ return Map.of("csv", (s, blockFactory) -> new CsvFormatReader(blockFactory));
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReader.java b/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReader.java
new file mode 100644
index 0000000000000..b4a0c9ae1e2eb
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-csv/src/main/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReader.java
@@ -0,0 +1,423 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.csv;
+
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvParser;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReader;
+import org.elasticsearch.xpack.esql.datasources.spi.SimpleSourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.SourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Simple CSV format reader for external datasources.
+ *
+ * CSV Format:
+ * - First line: schema definition (column_name:type_name,...)
+ * - Subsequent lines: data rows
+ * - Empty values are treated as null
+ * - Lines starting with "//" are comments and ignored
+ *
+ *
Supported types: integer, long, double, keyword, text, boolean, datetime
+ *
+ *
This reader works with any StorageProvider (HTTP, S3, local).
+ */
+public class CsvFormatReader implements FormatReader {
+
+ private final BlockFactory blockFactory;
+
+ public CsvFormatReader(BlockFactory blockFactory) {
+ this.blockFactory = blockFactory;
+ }
+
+ @Override
+ public SourceMetadata metadata(StorageObject object) throws IOException {
+ List schema = readSchema(object);
+ StoragePath objectPath = object.path();
+ return new SimpleSourceMetadata(schema, formatName(), objectPath.toString());
+ }
+
+ private List readSchema(StorageObject object) throws IOException {
+ try (
+ InputStream stream = object.newStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))
+ ) {
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.isEmpty() || line.startsWith("//")) {
+ continue;
+ }
+ // First non-comment line is the schema
+ return parseSchema(line);
+ }
+ throw new IOException("CSV file has no schema line");
+ }
+ }
+
+ @Override
+ public CloseableIterator read(StorageObject object, List projectedColumns, int batchSize) throws IOException {
+ InputStream stream = object.newStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
+
+ return new CsvBatchIterator(reader, stream, projectedColumns, batchSize);
+ }
+
+ @Override
+ public String formatName() {
+ return "csv";
+ }
+
+ @Override
+ public List fileExtensions() {
+ return List.of(".csv", ".tsv");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to close at reader level
+ }
+
+ private List parseSchema(String schemaLine) {
+ String[] columns = schemaLine.split(",");
+ List attributes = new ArrayList<>(columns.length);
+
+ for (String column : columns) {
+ String trimmedColumn = column.trim();
+ String[] parts = trimmedColumn.split(":");
+ if (parts.length != 2) {
+ throw new ParsingException("Invalid CSV schema format: [{}]. Expected 'name:type'", column);
+ }
+
+ String name = parts[0].trim();
+ String trimmedType = parts[1].trim();
+ String typeName = trimmedType.toUpperCase(java.util.Locale.ROOT);
+ DataType dataType = parseDataType(typeName);
+
+ EsField field = new EsField(name, dataType, java.util.Map.of(), true, EsField.TimeSeriesFieldType.NONE);
+ attributes.add(new FieldAttribute(Source.EMPTY, name, field));
+ }
+
+ return attributes;
+ }
+
+ private DataType parseDataType(String typeName) {
+ return switch (typeName) {
+ case "INTEGER", "INT", "I" -> DataType.INTEGER;
+ case "LONG", "L" -> DataType.LONG;
+ case "DOUBLE", "D" -> DataType.DOUBLE;
+ case "KEYWORD", "K", "STRING", "S" -> DataType.KEYWORD;
+ case "TEXT", "TXT" -> DataType.TEXT;
+ case "BOOLEAN", "BOOL" -> DataType.BOOLEAN;
+ case "DATETIME", "DATE", "DT" -> DataType.DATETIME;
+ case "NULL", "N" -> DataType.NULL;
+ default -> throw EsqlIllegalArgumentException.illegalDataType(typeName);
+ };
+ }
+
+ /**
+ * Iterator that reads CSV data in batches and converts to ESQL Pages.
+ * Uses Jackson CSV parser for robust CSV parsing with proper quote and escape handling.
+ */
+ private class CsvBatchIterator implements CloseableIterator {
+ private final BufferedReader reader;
+ private final InputStream stream;
+ private final List projectedColumns;
+ private final int batchSize;
+ private final CsvMapper csvMapper;
+
+ private List schema;
+ private List projectedIndices;
+ private Iterator> csvIterator;
+ private Page nextPage;
+ private boolean closed = false;
+
+ CsvBatchIterator(BufferedReader reader, InputStream stream, List projectedColumns, int batchSize) {
+ this.reader = reader;
+ this.stream = stream;
+ this.projectedColumns = projectedColumns;
+ this.batchSize = batchSize;
+ this.csvMapper = new CsvMapper();
+ this.csvMapper.enable(CsvParser.Feature.TRIM_SPACES);
+ this.csvMapper.enable(CsvParser.Feature.SKIP_EMPTY_LINES);
+ this.csvMapper.enable(CsvParser.Feature.WRAP_AS_ARRAY);
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (closed) {
+ return false;
+ }
+ if (nextPage != null) {
+ return true;
+ }
+ try {
+ nextPage = readNextBatch();
+ return nextPage != null;
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read CSV batch", e);
+ }
+ }
+
+ @Override
+ public Page next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+ Page result = nextPage;
+ nextPage = null;
+ return result;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ closed = true;
+ reader.close();
+ stream.close();
+ }
+ }
+
+ private Page readNextBatch() throws IOException {
+ if (schema == null) {
+ // Read schema from first non-comment line
+ String line;
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.isEmpty() || line.startsWith("//")) {
+ continue;
+ }
+ schema = parseSchema(line);
+ projectedIndices = computeProjectedIndices();
+
+ // Initialize CSV iterator with Jackson CSV parser
+ // Use WRAP_AS_ARRAY to read CSV rows as lists without predefined schema
+ CsvSchema csvSchema = CsvSchema.emptySchema()
+ .withColumnSeparator(',')
+ .withQuoteChar('"')
+ .withEscapeChar('\\')
+ .withNullValue("");
+
+ csvIterator = csvMapper.readerFor(List.class).with(csvSchema).readValues(reader);
+ break;
+ }
+ if (schema == null) {
+ return null; // No schema found
+ }
+ }
+
+ // Read batch of rows using Jackson CSV parser
+ List rows = new ArrayList<>();
+ while (rows.size() < batchSize && csvIterator.hasNext()) {
+ List rowList = csvIterator.next();
+ // Convert List to String array
+ String[] row = new String[rowList.size()];
+ for (int i = 0; i < rowList.size(); i++) {
+ Object val = rowList.get(i);
+ row[i] = val != null ? val.toString() : null;
+ }
+ // Skip comment lines (Jackson doesn't have native comment support)
+ if (row.length > 0) {
+ String firstCell = row[0];
+ if (firstCell != null) {
+ String trimmedFirstCell = firstCell.trim();
+ if (trimmedFirstCell.startsWith("//")) {
+ continue;
+ }
+ }
+ }
+ rows.add(row);
+ }
+
+ if (rows.isEmpty()) {
+ return null; // No more data
+ }
+
+ return convertRowsToPage(rows);
+ }
+
+ private List computeProjectedIndices() {
+ if (projectedColumns == null || projectedColumns.isEmpty()) {
+ // Return all columns
+ List indices = new ArrayList<>(schema.size());
+ for (int i = 0; i < schema.size(); i++) {
+ indices.add(i);
+ }
+ return indices;
+ }
+
+ // Map projected column names to indices
+ List indices = new ArrayList<>(projectedColumns.size());
+ for (String colName : projectedColumns) {
+ int index = -1;
+ for (int i = 0; i < schema.size(); i++) {
+ Attribute attr = schema.get(i);
+ if (attr.name().equals(colName)) {
+ index = i;
+ break;
+ }
+ }
+ if (index == -1) {
+ throw new EsqlIllegalArgumentException("Column not found in CSV schema: [{}]", colName);
+ }
+ indices.add(index);
+ }
+ return indices;
+ }
+
+ private Page convertRowsToPage(List rows) {
+ int rowCount = rows.size();
+ int columnCount = projectedIndices.size();
+
+ // Create block builders for projected columns
+ BlockUtils.BuilderWrapper[] builders = new BlockUtils.BuilderWrapper[columnCount];
+ try {
+ for (int i = 0; i < columnCount; i++) {
+ int schemaIndex = projectedIndices.get(i);
+ Attribute attr = schema.get(schemaIndex);
+ builders[i] = BlockUtils.wrapperFor(
+ blockFactory,
+ org.elasticsearch.compute.data.ElementType.fromJava(javaClassForDataType(attr.dataType())),
+ rowCount
+ );
+ }
+
+ // Fill blocks with data
+ for (String[] row : rows) {
+ // Jackson CSV may return shorter arrays if trailing values are empty
+ // We need to handle this gracefully
+ if (row.length > schema.size()) {
+ throw new ParsingException("CSV row has [{}] columns but schema defines [{}] columns", row.length, schema.size());
+ }
+
+ for (int i = 0; i < columnCount; i++) {
+ int schemaIndex = projectedIndices.get(i);
+ Attribute attr = schema.get(schemaIndex);
+
+ // Handle case where row is shorter than expected (trailing empty values)
+ String value = schemaIndex < row.length ? row[schemaIndex] : "";
+ if (value != null) {
+ value = value.trim();
+ }
+
+ Object converted = convertValue(value, attr.dataType());
+ BlockUtils.BuilderWrapper wrapper = builders[i];
+ wrapper.append().accept(converted);
+ }
+ }
+
+ // Build blocks
+ Block[] blocks = new Block[columnCount];
+ for (int i = 0; i < columnCount; i++) {
+ BlockUtils.BuilderWrapper wrapper = builders[i];
+ Block.Builder builder = wrapper.builder();
+ blocks[i] = builder.build();
+ }
+
+ return new Page(rowCount, blocks);
+ } finally {
+ Releasables.closeExpectNoException(builders);
+ }
+ }
+
+ private Class javaClassForDataType(DataType dataType) {
+ return switch (dataType) {
+ case INTEGER -> Integer.class;
+ case LONG, DATETIME -> Long.class;
+ case DOUBLE -> Double.class;
+ case KEYWORD, TEXT -> BytesRef.class;
+ case BOOLEAN -> Boolean.class;
+ case NULL -> Void.class;
+ default -> throw new IllegalArgumentException("Unsupported data type: " + dataType);
+ };
+ }
+
+ private Object convertValue(String value, DataType dataType) {
+ // Jackson CSV uses null for empty values when configured with withNullValue("")
+ // Also handle explicit "null" string
+ if (value == null || value.isEmpty() || value.equalsIgnoreCase("null")) {
+ return null;
+ }
+
+ try {
+ return switch (dataType) {
+ case INTEGER -> Integer.parseInt(value);
+ case LONG -> Long.parseLong(value);
+ case DOUBLE -> Double.parseDouble(value);
+ case KEYWORD, TEXT -> new BytesRef(value);
+ case BOOLEAN -> Booleans.parseBoolean(value);
+ case DATETIME -> parseDatetime(value);
+ case NULL -> null;
+ default -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
+ };
+ } catch (NumberFormatException e) {
+ throw new EsqlIllegalArgumentException(e, "Failed to parse CSV value [{}] as [{}]", value, dataType);
+ }
+ }
+
+ private long parseDatetime(String value) {
+ // Numeric strings (epoch millis) contain only digits and optionally a leading minus
+ if (looksNumeric(value)) {
+ try {
+ return Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ // overflow or not actually numeric, fall through to ISO-8601
+ }
+ }
+ try {
+ return Instant.parse(value).toEpochMilli();
+ } catch (DateTimeParseException e) {
+ throw new EsqlIllegalArgumentException(e, "Failed to parse CSV datetime value [{}]", value);
+ }
+ }
+
+ private static boolean looksNumeric(String value) {
+ int start = (value.charAt(0) == '-') ? 1 : 0;
+ if (start >= value.length()) {
+ return false;
+ }
+ for (int i = start; i < value.length(); i++) {
+ if (value.charAt(i) < '0' || value.charAt(i) > '9') {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-csv/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin b/x-pack/plugin/esql-datasource-csv/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
new file mode 100644
index 0000000000000..1edf44773d3d0
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-csv/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
@@ -0,0 +1 @@
+org.elasticsearch.xpack.esql.datasource.csv.CsvDataSourcePlugin
diff --git a/x-pack/plugin/esql-datasource-csv/src/test/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReaderTests.java b/x-pack/plugin/esql-datasource-csv/src/test/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReaderTests.java
new file mode 100644
index 0000000000000..6d1a12b0e5c28
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-csv/src/test/java/org/elasticsearch/xpack/esql/datasource/csv/CsvFormatReaderTests.java
@@ -0,0 +1,346 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.csv;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.elasticsearch.xpack.esql.parser.ParsingException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.List;
+
+public class CsvFormatReaderTests extends ESTestCase {
+
+ private BlockFactory blockFactory;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ blockFactory = BlockFactory.getInstance(new NoopCircuitBreaker("test-noop"), BigArrays.NON_RECYCLING_INSTANCE);
+ }
+
+ public void testSchema() throws IOException {
+ String csv = """
+ id:long,name:keyword,age:integer,active:boolean
+ 1,Alice,30,true
+ 2,Bob,25,false
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ List schema = reader.schema(object);
+
+ assertEquals(4, schema.size());
+ assertEquals("id", schema.get(0).name());
+ assertEquals(DataType.LONG, schema.get(0).dataType());
+ assertEquals("name", schema.get(1).name());
+ assertEquals(DataType.KEYWORD, schema.get(1).dataType());
+ assertEquals("age", schema.get(2).name());
+ assertEquals(DataType.INTEGER, schema.get(2).dataType());
+ assertEquals("active", schema.get(3).name());
+ assertEquals(DataType.BOOLEAN, schema.get(3).dataType());
+ }
+
+ public void testSchemaWithComments() throws IOException {
+ String csv = """
+ // This is a comment
+ // Another comment
+ id:long,name:keyword
+ 1,Alice
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ List schema = reader.schema(object);
+
+ assertEquals(2, schema.size());
+ assertEquals("id", schema.get(0).name());
+ assertEquals("name", schema.get(1).name());
+ }
+
+ public void testReadAllColumns() throws IOException {
+ String csv = """
+ id:long,name:keyword,score:double
+ 1,Alice,95.5
+ 2,Bob,87.3
+ 3,Charlie,92.1
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(3, page.getPositionCount());
+ assertEquals(3, page.getBlockCount());
+
+ // Check first row
+ assertEquals(1L, ((LongBlock) page.getBlock(0)).getLong(0));
+ assertEquals(new BytesRef("Alice"), ((BytesRefBlock) page.getBlock(1)).getBytesRef(0, new BytesRef()));
+ assertEquals(95.5, ((DoubleBlock) page.getBlock(2)).getDouble(0), 0.001);
+
+ // Check second row
+ assertEquals(2L, ((LongBlock) page.getBlock(0)).getLong(1));
+ assertEquals(new BytesRef("Bob"), ((BytesRefBlock) page.getBlock(1)).getBytesRef(1, new BytesRef()));
+ assertEquals(87.3, ((DoubleBlock) page.getBlock(2)).getDouble(1), 0.001);
+
+ assertFalse(iterator.hasNext());
+ }
+ }
+
+ public void testReadProjectedColumns() throws IOException {
+ String csv = """
+ id:long,name:keyword,score:double
+ 1,Alice,95.5
+ 2,Bob,87.3
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ // Project only name and score
+ try (CloseableIterator iterator = reader.read(object, List.of("name", "score"), 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(2, page.getPositionCount());
+ assertEquals(2, page.getBlockCount()); // Only 2 projected columns
+
+ assertEquals(new BytesRef("Alice"), ((BytesRefBlock) page.getBlock(0)).getBytesRef(0, new BytesRef()));
+ assertEquals(95.5, ((DoubleBlock) page.getBlock(1)).getDouble(0), 0.001);
+ }
+ }
+
+ public void testReadWithBatching() throws IOException {
+ StringBuilder csv = new StringBuilder("id:long,value:integer\n");
+ for (int i = 1; i <= 25; i++) {
+ csv.append(i).append(",").append(i * 10).append("\n");
+ }
+
+ StorageObject object = createStorageObject(csv.toString());
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ int batchSize = 10;
+ int totalRows = 0;
+
+ try (CloseableIterator iterator = reader.read(object, null, batchSize)) {
+ // First batch: 10 rows
+ assertTrue(iterator.hasNext());
+ Page page1 = iterator.next();
+ assertEquals(10, page1.getPositionCount());
+ totalRows += page1.getPositionCount();
+
+ // Second batch: 10 rows
+ assertTrue(iterator.hasNext());
+ Page page2 = iterator.next();
+ assertEquals(10, page2.getPositionCount());
+ totalRows += page2.getPositionCount();
+
+ // Third batch: 5 rows
+ assertTrue(iterator.hasNext());
+ Page page3 = iterator.next();
+ assertEquals(5, page3.getPositionCount());
+ totalRows += page3.getPositionCount();
+
+ assertFalse(iterator.hasNext());
+ }
+
+ assertEquals(25, totalRows);
+ }
+
+ public void testReadWithNullValues() throws IOException {
+ String csv = """
+ id:long,name:keyword,score:double
+ 1,Alice,95.5
+ 2,,87.3
+ 3,Charlie,
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(3, page.getPositionCount());
+
+ // First row: all values present
+ assertFalse(page.getBlock(0).isNull(0));
+ assertFalse(page.getBlock(1).isNull(0));
+ assertFalse(page.getBlock(2).isNull(0));
+
+ // Second row: name is null
+ assertFalse(page.getBlock(0).isNull(1));
+ assertTrue(page.getBlock(1).isNull(1));
+ assertFalse(page.getBlock(2).isNull(1));
+
+ // Third row: score is null
+ assertFalse(page.getBlock(0).isNull(2));
+ assertFalse(page.getBlock(1).isNull(2));
+ assertTrue(page.getBlock(2).isNull(2));
+ }
+ }
+
+ public void testReadWithCommentsInData() throws IOException {
+ String csv = """
+ id:long,name:keyword
+ // This is a comment
+ 1,Alice
+ // Another comment
+ 2,Bob
+ """;
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ // Comments should be skipped, only 2 data rows
+ assertEquals(2, page.getPositionCount());
+ assertEquals(1L, ((LongBlock) page.getBlock(0)).getLong(0));
+ assertEquals(2L, ((LongBlock) page.getBlock(0)).getLong(1));
+ }
+ }
+
+ public void testFormatName() {
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+ assertEquals("csv", reader.formatName());
+ }
+
+ public void testFileExtensions() {
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+ List extensions = reader.fileExtensions();
+ assertEquals(2, extensions.size());
+ assertTrue(extensions.contains(".csv"));
+ assertTrue(extensions.contains(".tsv"));
+ }
+
+ public void testInvalidSchema() {
+ String csv = "invalid_schema_no_colon\n";
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ ParsingException e = expectThrows(ParsingException.class, () -> reader.schema(object));
+ assertTrue(e.getMessage().contains("Invalid CSV schema format"));
+ }
+
+ public void testReadDatetimeEpochMillis() throws IOException {
+ long epochMillis = 1609459200000L; // 2021-01-01T00:00:00.000Z
+ String csv = "id:long,ts:datetime\n1," + epochMillis + "\n";
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+ assertEquals(1, page.getPositionCount());
+ assertEquals(epochMillis, ((LongBlock) page.getBlock(1)).getLong(0));
+ }
+ }
+
+ public void testReadDatetimeIso8601() throws IOException {
+ String csv = "id:long,ts:datetime\n1,1953-09-02T00:00:00.000Z\n2,2021-01-01T00:00:00Z\n";
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+ assertEquals(2, page.getPositionCount());
+ assertEquals(Instant.parse("1953-09-02T00:00:00.000Z").toEpochMilli(), ((LongBlock) page.getBlock(1)).getLong(0));
+ assertEquals(Instant.parse("2021-01-01T00:00:00Z").toEpochMilli(), ((LongBlock) page.getBlock(1)).getLong(1));
+ }
+ }
+
+ public void testReadDatetimeMixed() throws IOException {
+ long epochMillis = 1609459200000L; // 2021-01-01T00:00:00.000Z
+ String csv = "id:long,ts:datetime\n1," + epochMillis + "\n2,1953-09-02T00:00:00.000Z\n";
+
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(object, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+ assertEquals(2, page.getPositionCount());
+ assertEquals(epochMillis, ((LongBlock) page.getBlock(1)).getLong(0));
+ assertEquals(Instant.parse("1953-09-02T00:00:00.000Z").toEpochMilli(), ((LongBlock) page.getBlock(1)).getLong(1));
+ }
+ }
+
+ public void testUnsupportedType() {
+ String csv = "id:unsupported_type\n";
+ StorageObject object = createStorageObject(csv);
+ CsvFormatReader reader = new CsvFormatReader(blockFactory);
+
+ EsqlIllegalArgumentException e = expectThrows(EsqlIllegalArgumentException.class, () -> reader.schema(object));
+ assertTrue(e.getMessage().contains("illegal data type"));
+ }
+
+ private StorageObject createStorageObject(String csvContent) {
+ byte[] bytes = csvContent.getBytes(StandardCharsets.UTF_8);
+
+ return new StorageObject() {
+ @Override
+ public InputStream newStream() throws IOException {
+ return new ByteArrayInputStream(bytes);
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ throw new UnsupportedOperationException("Range reads not needed for CSV");
+ }
+
+ @Override
+ public long length() throws IOException {
+ return bytes.length;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ return Instant.now();
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ return true;
+ }
+
+ @Override
+ public StoragePath path() {
+ return StoragePath.of("memory://test.csv");
+ }
+ };
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/build.gradle b/x-pack/plugin/esql-datasource-http/build.gradle
new file mode 100644
index 0000000000000..aefc2f392b5a1
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/build.gradle
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.publish'
+
+esplugin {
+ name = 'esql-datasource-http'
+ description = 'HTTP/HTTPS and local file storage providers for ESQL external data sources'
+ classname = 'org.elasticsearch.xpack.esql.datasource.http.HttpDataSourcePlugin'
+ extendedPlugins = ['x-pack-esql']
+}
+
+base {
+ archivesName = 'esql-datasource-http'
+}
+
+dependencies {
+ // SPI interfaces from ESQL core
+ compileOnly project(path: xpackModule('esql'))
+ compileOnly project(path: xpackModule('esql-core'))
+ compileOnly project(path: xpackModule('core'))
+ compileOnly project(':server')
+ compileOnly project(xpackModule('esql:compute'))
+
+ testImplementation project(':test:framework')
+ testImplementation(testArtifact(project(xpackModule('core'))))
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpConfiguration.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpConfiguration.java
new file mode 100644
index 0000000000000..95c3217d2abb9
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpConfiguration.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Configuration for HTTP/HTTPS storage access.
+ * Provides settings for timeouts, redirects, and custom headers.
+ */
+public final class HttpConfiguration {
+ private final Duration connectTimeout;
+ private final Duration requestTimeout;
+ private final boolean followRedirects;
+ private final Map customHeaders;
+ private final int maxRetries;
+
+ /**
+ * Creates a new HttpConfiguration with default settings.
+ */
+ public static HttpConfiguration defaults() {
+ return new Builder().build();
+ }
+
+ /**
+ * Creates a new builder for HttpConfiguration.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private HttpConfiguration(Builder builder) {
+ if (builder.connectTimeout == null) {
+ throw new IllegalArgumentException("connectTimeout cannot be null");
+ }
+ if (builder.requestTimeout == null) {
+ throw new IllegalArgumentException("requestTimeout cannot be null");
+ }
+ if (builder.customHeaders == null) {
+ throw new IllegalArgumentException("customHeaders cannot be null");
+ }
+ this.connectTimeout = builder.connectTimeout;
+ this.requestTimeout = builder.requestTimeout;
+ this.followRedirects = builder.followRedirects;
+ this.customHeaders = Map.copyOf(builder.customHeaders);
+ this.maxRetries = builder.maxRetries;
+ }
+
+ public Duration connectTimeout() {
+ return connectTimeout;
+ }
+
+ public Duration requestTimeout() {
+ return requestTimeout;
+ }
+
+ public boolean followRedirects() {
+ return followRedirects;
+ }
+
+ public Map customHeaders() {
+ return customHeaders;
+ }
+
+ public int maxRetries() {
+ return maxRetries;
+ }
+
+ public static final class Builder {
+ private Duration connectTimeout = Duration.ofSeconds(30);
+ private Duration requestTimeout = Duration.ofMinutes(5);
+ private boolean followRedirects = true;
+ private Map customHeaders = Map.of();
+ private int maxRetries = 3;
+
+ private Builder() {}
+
+ public Builder connectTimeout(Duration connectTimeout) {
+ if (connectTimeout == null) {
+ throw new IllegalArgumentException("connectTimeout cannot be null");
+ }
+ this.connectTimeout = connectTimeout;
+ return this;
+ }
+
+ public Builder requestTimeout(Duration requestTimeout) {
+ if (requestTimeout == null) {
+ throw new IllegalArgumentException("requestTimeout cannot be null");
+ }
+ this.requestTimeout = requestTimeout;
+ return this;
+ }
+
+ public Builder followRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ return this;
+ }
+
+ public Builder customHeaders(Map customHeaders) {
+ if (customHeaders == null) {
+ throw new IllegalArgumentException("customHeaders cannot be null");
+ }
+ this.customHeaders = customHeaders;
+ return this;
+ }
+
+ public Builder maxRetries(int maxRetries) {
+ if (maxRetries < 0) {
+ throw new IllegalArgumentException("maxRetries must be non-negative");
+ }
+ this.maxRetries = maxRetries;
+ return this;
+ }
+
+ public HttpConfiguration build() {
+ return new HttpConfiguration(this);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ HttpConfiguration that = (HttpConfiguration) o;
+ return followRedirects == that.followRedirects
+ && maxRetries == that.maxRetries
+ && Objects.equals(connectTimeout, that.connectTimeout)
+ && Objects.equals(requestTimeout, that.requestTimeout)
+ && Objects.equals(customHeaders, that.customHeaders);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(connectTimeout, requestTimeout, followRedirects, customHeaders, maxRetries);
+ }
+
+ @Override
+ public String toString() {
+ return "HttpConfiguration{"
+ + "connectTimeout="
+ + connectTimeout
+ + ", requestTimeout="
+ + requestTimeout
+ + ", followRedirects="
+ + followRedirects
+ + ", customHeaders="
+ + customHeaders
+ + ", maxRetries="
+ + maxRetries
+ + '}';
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpDataSourcePlugin.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpDataSourcePlugin.java
new file mode 100644
index 0000000000000..178a2634c2044
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpDataSourcePlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.esql.datasource.http.local.LocalStorageProvider;
+import org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProviderFactory;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Data source plugin that provides HTTP/HTTPS and local file storage providers
+ * for ESQL external data sources.
+ *
+ * This plugin provides:
+ *
+ * HTTP/HTTPS storage provider for reading from web servers
+ * Local file system storage provider for testing and development
+ *
+ *
+ * These implementations have no heavy external dependencies and use JDK's
+ * built-in {@code HttpClient} and {@code java.nio} APIs.
+ *
+ *
The executor for async HTTP I/O is injected via the
+ * {@link DataSourcePlugin#storageProviders(Settings, ExecutorService)} SPI method,
+ * backed by the ES GENERIC thread pool.
+ */
+public class HttpDataSourcePlugin extends Plugin implements DataSourcePlugin {
+
+ @Override
+ public Map storageProviders(Settings settings, ExecutorService executor) {
+ return Map.of(
+ "http",
+ s -> new HttpStorageProvider(HttpConfiguration.defaults(), executor),
+ "https",
+ s -> new HttpStorageProvider(HttpConfiguration.defaults(), executor),
+ "file",
+ s -> new LocalStorageProvider()
+ );
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObject.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObject.java
new file mode 100644
index 0000000000000..d022e9376ca85
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObject.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.concurrent.Executor;
+
+/**
+ * StorageObject implementation using HTTP Range requests for efficient partial reads.
+ * Uses standard Java HttpClient and InputStream - no custom stream classes needed.
+ *
+ * Supports:
+ *
+ * Full object reads via GET
+ * Range reads via HTTP Range header for columnar formats
+ * Metadata retrieval via HEAD requests
+ *
+ */
+public final class HttpStorageObject implements StorageObject {
+
+ private final HttpClient client;
+ private final StoragePath path;
+ private final URI uri; // Cached URI to avoid repeated parsing
+ private final HttpConfiguration config;
+
+ // Cached metadata to avoid repeated HEAD requests
+ private Long cachedLength;
+ private Instant cachedLastModified;
+ private Boolean cachedExists;
+
+ /**
+ * Creates an HttpStorageObject without pre-known metadata.
+ */
+ public HttpStorageObject(HttpClient client, StoragePath path, HttpConfiguration config) {
+ if (client == null) {
+ throw new IllegalArgumentException("client cannot be null");
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("path cannot be null");
+ }
+ if (config == null) {
+ throw new IllegalArgumentException("config cannot be null");
+ }
+ this.client = client;
+ this.path = path;
+ this.uri = URI.create(path.toString());
+ this.config = config;
+ }
+
+ /**
+ * Creates an HttpStorageObject with pre-known length.
+ */
+ public HttpStorageObject(HttpClient client, StoragePath path, HttpConfiguration config, long length) {
+ this(client, path, config);
+ this.cachedLength = length;
+ }
+
+ /**
+ * Creates an HttpStorageObject with pre-known length and last modified time.
+ */
+ public HttpStorageObject(HttpClient client, StoragePath path, HttpConfiguration config, long length, Instant lastModified) {
+ this(client, path, config, length);
+ this.cachedLastModified = lastModified;
+ }
+
+ @Override
+ public InputStream newStream() throws IOException {
+ return sendRequest(this::buildGetRequest, HttpResponse.BodyHandlers.ofInputStream(), response -> {
+ int statusCode = response.statusCode();
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("Failed to read object from " + path + ", HTTP status: " + statusCode);
+ }
+ return response.body();
+ });
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ if (position < 0) {
+ throw new IllegalArgumentException("position must be non-negative, got: " + position);
+ }
+ if (length < 0) {
+ throw new IllegalArgumentException("length must be non-negative, got: " + length);
+ }
+
+ return sendRequest(() -> buildRangeRequest(position, length), HttpResponse.BodyHandlers.ofInputStream(), response -> {
+ int statusCode = response.statusCode();
+ // 206 = Partial Content (successful range request)
+ // 200 = OK (server doesn't support ranges but returned full content)
+ if (statusCode == HttpStatus.SC_PARTIAL_CONTENT) {
+ return response.body();
+ } else if (statusCode == HttpStatus.SC_OK) {
+ // Server doesn't support Range requests, skip to position manually
+ InputStream stream = response.body();
+ long skipped = stream.skip(position);
+ if (skipped != position) {
+ stream.close();
+ throw new IOException("Failed to skip to position " + position + ", only skipped " + skipped + " bytes");
+ }
+ // Wrap in a limited stream to ensure we only read 'length' bytes
+ return new BoundedInputStream(stream, length);
+ } else {
+ throw new IOException("Range request failed for " + path + ", HTTP status: " + statusCode);
+ }
+ });
+ }
+
+ @Override
+ public long length() throws IOException {
+ if (cachedLength == null) {
+ fetchMetadata();
+ }
+ return cachedLength;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ if (cachedLastModified == null) {
+ fetchMetadata();
+ }
+ return cachedLastModified;
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ if (cachedExists == null) {
+ fetchMetadata();
+ }
+ return cachedExists;
+ }
+
+ @Override
+ public StoragePath path() {
+ return path;
+ }
+
+ // === ASYNC API (native implementation using HttpClient.sendAsync) ===
+
+ /**
+ * Async byte read using HttpClient.sendAsync() for native non-blocking I/O.
+ *
+ * This implementation uses Java's built-in async HTTP client to avoid blocking
+ * threads during I/O. The executor parameter is ignored since HttpClient manages
+ * its own thread pool for async operations (configured at client creation time).
+ *
+ * @param position the starting byte position
+ * @param length the number of bytes to read
+ * @param executor executor (unused - HttpClient uses executor configured at creation)
+ * @param listener callback for the result or failure
+ */
+ @Override
+ public void readBytesAsync(long position, long length, Executor executor, ActionListener listener) {
+ if (position < 0) {
+ listener.onFailure(new IllegalArgumentException("position must be non-negative, got: " + position));
+ return;
+ }
+ if (length < 0) {
+ listener.onFailure(new IllegalArgumentException("length must be non-negative, got: " + length));
+ return;
+ }
+
+ HttpRequest request = buildRangeRequest(position, length);
+
+ // Use native async HTTP - no blocking, no extra threads needed
+ client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).whenComplete((response, throwable) -> {
+ if (throwable != null) {
+ listener.onFailure(throwable instanceof Exception ex ? ex : new RuntimeException(throwable));
+ return;
+ }
+
+ int statusCode = response.statusCode();
+ // 206 = Partial Content (successful range request)
+ // 200 = OK (server doesn't support ranges but returned full content - need to slice)
+ if (statusCode == HttpStatus.SC_PARTIAL_CONTENT) {
+ listener.onResponse(ByteBuffer.wrap(response.body()));
+ } else if (statusCode == HttpStatus.SC_OK) {
+ // Server doesn't support Range requests, slice the response
+ byte[] fullBody = response.body();
+ int bodyLength = fullBody.length;
+ if (position >= bodyLength) {
+ listener.onFailure(
+ new IOException("Position " + position + " is beyond content length " + bodyLength + " for " + path)
+ );
+ return;
+ }
+ int actualLength = (int) Math.min(length, bodyLength - position);
+ byte[] slice = new byte[actualLength];
+ System.arraycopy(fullBody, (int) position, slice, 0, actualLength);
+ listener.onResponse(ByteBuffer.wrap(slice));
+ } else {
+ listener.onFailure(new IOException("Range request failed for " + path + ", HTTP status: " + statusCode));
+ }
+ });
+ }
+
+ /**
+ * Returns true - HttpStorageObject has native async support via HttpClient.sendAsync().
+ */
+ @Override
+ public boolean supportsNativeAsync() {
+ return true;
+ }
+
+ // === Private helper methods ===
+
+ /**
+ * Builds a simple GET request without Range header.
+ */
+ private HttpRequest buildGetRequest() {
+ HttpRequest.Builder builder = HttpRequest.newBuilder().uri(uri).GET().timeout(config.requestTimeout());
+ addCustomHeaders(builder);
+ return builder.build();
+ }
+
+ /**
+ * Builds a GET request with Range header for partial content.
+ */
+ private HttpRequest buildRangeRequest(long position, long length) {
+ // HTTP Range uses inclusive end: "bytes=start-end"
+ long endPosition = position + length - 1;
+ String rangeValue = "bytes=" + position + "-" + endPosition;
+
+ HttpRequest.Builder builder = HttpRequest.newBuilder()
+ .uri(uri)
+ .header(HttpHeaders.RANGE, rangeValue)
+ .GET()
+ .timeout(config.requestTimeout());
+ addCustomHeaders(builder);
+ return builder.build();
+ }
+
+ /**
+ * Builds a HEAD request for metadata retrieval.
+ */
+ private HttpRequest buildHeadRequest() {
+ HttpRequest.Builder builder = HttpRequest.newBuilder()
+ .uri(uri)
+ .method("HEAD", HttpRequest.BodyPublishers.noBody())
+ .timeout(config.requestTimeout());
+ addCustomHeaders(builder);
+ return builder.build();
+ }
+
+ /**
+ * Adds custom headers from configuration to the request builder.
+ */
+ private void addCustomHeaders(HttpRequest.Builder builder) {
+ Map headers = config.customHeaders();
+ for (Map.Entry entry : headers.entrySet()) {
+ builder.header(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Sends a synchronous HTTP request with proper interrupt handling.
+ *
+ * This method centralizes the try/catch for InterruptedException, ensuring:
+ *
+ * The interrupt flag is restored via Thread.currentThread().interrupt()
+ * The exception is wrapped in IOException to match the interface contract
+ *
+ *
+ * @param requestSupplier supplies the HTTP request to send
+ * @param bodyHandler handles the response body
+ * @param responseHandler processes the response and returns the result
+ * @return the result from responseHandler
+ * @throws IOException on I/O errors or if interrupted
+ */
+ private R sendRequest(
+ CheckedFunction requestSupplier,
+ HttpResponse.BodyHandler bodyHandler,
+ CheckedFunction, R, IOException> responseHandler
+ ) throws IOException {
+ HttpRequest request = requestSupplier.apply(null);
+ try {
+ HttpResponse response = client.send(request, bodyHandler);
+ return responseHandler.apply(response);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("HTTP request interrupted for " + path, e);
+ }
+ }
+
+ /**
+ * Overload for request suppliers that don't throw.
+ */
+ @FunctionalInterface
+ private interface RequestSupplier {
+ HttpRequest get();
+ }
+
+ private R sendRequest(
+ RequestSupplier requestSupplier,
+ HttpResponse.BodyHandler bodyHandler,
+ CheckedFunction, R, IOException> responseHandler
+ ) throws IOException {
+ HttpRequest request = requestSupplier.get();
+ try {
+ HttpResponse response = client.send(request, bodyHandler);
+ return responseHandler.apply(response);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("HTTP request interrupted for " + path, e);
+ }
+ }
+
+ /**
+ * Fetches metadata via HEAD request and caches the results.
+ */
+ private void fetchMetadata() throws IOException {
+ sendRequest(this::buildHeadRequest, HttpResponse.BodyHandlers.discarding(), response -> {
+ int statusCode = response.statusCode();
+ if (statusCode == HttpStatus.SC_OK) {
+ cachedExists = true;
+
+ // Extract Content-Length
+ OptionalLong contentLength = response.headers().firstValueAsLong(HttpHeaders.CONTENT_LENGTH);
+ if (contentLength.isPresent() == false) {
+ throw new IOException("Server did not return " + HttpHeaders.CONTENT_LENGTH + " for " + path);
+ }
+ cachedLength = contentLength.getAsLong();
+
+ // Extract Last-Modified (optional)
+ java.util.Optional lastModified = response.headers().firstValue(HttpHeaders.LAST_MODIFIED);
+ cachedLastModified = lastModified.isPresent() ? parseHttpDate(lastModified.get()) : null;
+ } else if (statusCode == HttpStatus.SC_NOT_FOUND) {
+ cachedExists = false;
+ cachedLength = 0L;
+ cachedLastModified = null;
+ } else {
+ throw new IOException("HEAD request failed for " + path + ", HTTP status: " + statusCode);
+ }
+ return null; // Void return
+ });
+ }
+
+ /**
+ * Parses HTTP date format (RFC 1123).
+ * Example: "Wed, 21 Oct 2015 07:28:00 GMT"
+ */
+ private Instant parseHttpDate(String dateString) {
+ try {
+ return ZonedDateTime.parse(dateString, DateTimeFormatter.RFC_1123_DATE_TIME).toInstant();
+ } catch (DateTimeParseException e) {
+ // If parsing fails, return null rather than throwing
+ return null;
+ }
+ }
+
+ /**
+ * InputStream wrapper that limits the number of bytes that can be read.
+ * Used when server doesn't support Range requests.
+ */
+ private static final class BoundedInputStream extends InputStream {
+ private final InputStream delegate;
+ private long remaining;
+
+ BoundedInputStream(InputStream delegate, long limit) {
+ this.delegate = delegate;
+ this.remaining = limit;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ int b = delegate.read();
+ if (b >= 0) {
+ remaining--;
+ }
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ int toRead = (int) Math.min(len, remaining);
+ int bytesRead = delegate.read(b, off, toRead);
+ if (bytesRead > 0) {
+ remaining -= bytesRead;
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProvider.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProvider.java
new file mode 100644
index 0000000000000..89c1e27903d51
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProvider.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import org.elasticsearch.xpack.esql.datasources.StorageIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProvider;
+
+import java.io.IOException;
+import java.net.http.HttpClient;
+import java.time.Instant;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * StorageProvider implementation for HTTP/HTTPS using Java's built-in HttpClient.
+ *
+ * Features:
+ * - Full object reads via GET
+ * - Range reads via HTTP Range header
+ * - Metadata retrieval via HEAD
+ * - Configurable timeouts and redirects
+ *
+ * Note: HTTP/HTTPS does not support directory listing, so listObjects() returns null.
+ */
+public final class HttpStorageProvider implements StorageProvider {
+ private final HttpClient httpClient;
+ private final HttpConfiguration config;
+
+ /**
+ * Creates an HttpStorageProvider with configuration and executor.
+ *
+ * @param config the HTTP configuration
+ * @param executor the executor service for async operations
+ */
+ public HttpStorageProvider(HttpConfiguration config, ExecutorService executor) {
+ if (config == null) {
+ throw new IllegalArgumentException("config cannot be null");
+ }
+ if (executor == null) {
+ throw new IllegalArgumentException("executor cannot be null");
+ }
+
+ this.config = config;
+ this.httpClient = HttpClient.newBuilder()
+ .connectTimeout(config.connectTimeout())
+ .followRedirects(config.followRedirects() ? HttpClient.Redirect.NORMAL : HttpClient.Redirect.NEVER)
+ .executor(executor)
+ .build();
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path) {
+ validateHttpScheme(path);
+ return new HttpStorageObject(httpClient, path, config);
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length) {
+ validateHttpScheme(path);
+ return new HttpStorageObject(httpClient, path, config, length);
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length, Instant lastModified) {
+ validateHttpScheme(path);
+ return new HttpStorageObject(httpClient, path, config, length, lastModified);
+ }
+
+ @Override
+ public StorageIterator listObjects(StoragePath prefix, boolean recursive) throws IOException {
+ throw new UnsupportedOperationException("HTTP does not support directory listing");
+ }
+
+ @Override
+ public boolean exists(StoragePath path) throws IOException {
+ validateHttpScheme(path);
+ StorageObject object = newObject(path);
+ return object.exists();
+ }
+
+ @Override
+ public List supportedSchemes() {
+ return List.of("http", "https");
+ }
+
+ @Override
+ public void close() {
+ // HttpClient implements AutoCloseable in Java 21+
+ // Closing it shuts down the internal selector thread and connection pool
+ httpClient.close();
+ }
+
+ private void validateHttpScheme(StoragePath path) {
+ String scheme = path.scheme().toLowerCase(Locale.ROOT);
+ if ("http".equals(scheme) == false && "https".equals(scheme) == false) {
+ throw new IllegalArgumentException("HttpStorageProvider only supports http:// and https:// schemes, got: " + scheme);
+ }
+ }
+
+ public HttpClient httpClient() {
+ return httpClient;
+ }
+
+ public HttpConfiguration config() {
+ return config;
+ }
+
+ @Override
+ public String toString() {
+ return "HttpStorageProvider{config=" + config + "}";
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageObject.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageObject.java
new file mode 100644
index 0000000000000..7fb5eb4f3b7c6
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageObject.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http.local;
+
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Instant;
+
+/**
+ * StorageObject implementation for local file system.
+ *
+ * Supports:
+ * - Full file reads via FileInputStream
+ * - Range reads via RandomAccessFile for columnar formats
+ * - File metadata (size, last modified)
+ */
+public final class LocalStorageObject implements StorageObject {
+ private final Path filePath;
+ private final StoragePath storagePath;
+
+ // Cached metadata to avoid repeated file system calls
+ private Long cachedLength;
+ private Instant cachedLastModified;
+ private Boolean cachedExists;
+
+ public LocalStorageObject(Path filePath) {
+ if (filePath == null) {
+ throw new IllegalArgumentException("filePath cannot be null");
+ }
+ this.filePath = filePath;
+ this.storagePath = StoragePath.of("file://" + filePath.toAbsolutePath());
+ }
+
+ public LocalStorageObject(Path filePath, long length) {
+ this(filePath);
+ this.cachedLength = length;
+ }
+
+ public LocalStorageObject(Path filePath, long length, Instant lastModified) {
+ this(filePath, length);
+ this.cachedLastModified = lastModified;
+ }
+
+ @Override
+ public InputStream newStream() throws IOException {
+ if (Files.exists(filePath) == false) {
+ throw new IOException("File does not exist: " + filePath);
+ }
+
+ if (Files.isRegularFile(filePath) == false) {
+ throw new IOException("Path is not a regular file: " + filePath);
+ }
+
+ return Files.newInputStream(filePath);
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ if (position < 0) {
+ throw new IllegalArgumentException("position must be non-negative, got: " + position);
+ }
+ if (length < 0) {
+ throw new IllegalArgumentException("length must be non-negative, got: " + length);
+ }
+
+ if (Files.exists(filePath) == false) {
+ throw new IOException("File does not exist: " + filePath);
+ }
+
+ if (Files.isRegularFile(filePath) == false) {
+ throw new IOException("Path is not a regular file: " + filePath);
+ }
+
+ // Use RandomAccessFile for efficient range reads
+ return new RangeInputStream(filePath, position, length);
+ }
+
+ @Override
+ public long length() throws IOException {
+ if (cachedLength == null) {
+ fetchMetadata();
+ }
+ return cachedLength;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ if (cachedLastModified == null) {
+ fetchMetadata();
+ }
+ return cachedLastModified;
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ if (cachedExists == null) {
+ fetchMetadata();
+ }
+ return cachedExists;
+ }
+
+ @Override
+ public StoragePath path() {
+ return storagePath;
+ }
+
+ private void fetchMetadata() throws IOException {
+ if (Files.exists(filePath)) {
+ cachedExists = true;
+ BasicFileAttributes attrs = Files.readAttributes(filePath, BasicFileAttributes.class);
+ cachedLength = attrs.size();
+ cachedLastModified = attrs.lastModifiedTime().toInstant();
+ } else {
+ cachedExists = false;
+ cachedLength = 0L;
+ cachedLastModified = null;
+ }
+ }
+
+ /**
+ * InputStream implementation for reading a specific range from a file.
+ * Uses FileChannel for efficient seeking and reading (avoids forbidden RandomAccessFile).
+ */
+ private static final class RangeInputStream extends InputStream {
+ private final FileChannel channel;
+ private final InputStream delegate;
+ private long remaining;
+
+ RangeInputStream(Path filePath, long position, long length) throws IOException {
+ this.remaining = length;
+ boolean success = false;
+ FileChannel ch = null;
+ try {
+ ch = FileChannel.open(filePath, StandardOpenOption.READ);
+ ch.position(position);
+ this.channel = ch;
+ this.delegate = Channels.newInputStream(ch);
+ success = true;
+ } finally {
+ if (success == false && ch != null) {
+ ch.close();
+ }
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ int b = delegate.read();
+ if (b >= 0) {
+ remaining--;
+ }
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ int toRead = (int) Math.min(len, remaining);
+ int bytesRead = delegate.read(b, off, toRead);
+ if (bytesRead > 0) {
+ remaining -= bytesRead;
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public void close() throws IOException {
+ channel.close();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (n <= 0) {
+ return 0;
+ }
+ long toSkip = Math.min(n, remaining);
+ long skipped = delegate.skip(toSkip);
+ remaining -= skipped;
+ return skipped;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return (int) Math.min(remaining, Integer.MAX_VALUE);
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProvider.java b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProvider.java
new file mode 100644
index 0000000000000..0c2791f9a886c
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProvider.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http.local;
+
+import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.xpack.esql.datasources.StorageEntry;
+import org.elasticsearch.xpack.esql.datasources.StorageIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProvider;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+/**
+ * StorageProvider implementation for local file system access.
+ *
+ * Features:
+ * - Full file reads
+ * - Range reads via RandomAccessFile
+ * - Directory listing
+ * - File metadata (size, last modified)
+ *
+ * This implementation is primarily for testing and development purposes.
+ */
+public final class LocalStorageProvider implements StorageProvider {
+
+ private static final String FILE_SCHEME_PREFIX = "file" + StoragePath.SCHEME_SEPARATOR;
+
+ /**
+ * Creates a LocalStorageProvider.
+ */
+ public LocalStorageProvider() {
+ // No configuration needed for local file system
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path) {
+ validateFileScheme(path);
+ return new LocalStorageObject(toFilePath(path));
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length) {
+ validateFileScheme(path);
+ return new LocalStorageObject(toFilePath(path), length);
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length, Instant lastModified) {
+ validateFileScheme(path);
+ return new LocalStorageObject(toFilePath(path), length, lastModified);
+ }
+
+ @Override
+ public StorageIterator listObjects(StoragePath prefix, boolean recursive) throws IOException {
+ validateFileScheme(prefix);
+ Path dirPath = toFilePath(prefix);
+
+ if (Files.exists(dirPath) == false) {
+ throw new IOException("Directory does not exist: " + dirPath);
+ }
+
+ if (Files.isDirectory(dirPath) == false) {
+ throw new IOException("Path is not a directory: " + dirPath);
+ }
+
+ return new LocalStorageIterator(dirPath, recursive);
+ }
+
+ @Override
+ public boolean exists(StoragePath path) throws IOException {
+ validateFileScheme(path);
+ Path filePath = toFilePath(path);
+ return Files.exists(filePath);
+ }
+
+ @Override
+ public List supportedSchemes() {
+ return List.of("file");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to clean up for local file system
+ }
+
+ /**
+ * Validates that the path uses the file:// scheme.
+ */
+ private void validateFileScheme(StoragePath path) {
+ String scheme = path.scheme().toLowerCase(Locale.ROOT);
+ if (scheme.equals("file") == false) {
+ throw new IllegalArgumentException("LocalStorageProvider only supports file:// scheme, got: " + scheme);
+ }
+ }
+
+ /**
+ * Converts a StoragePath to a java.nio.file.Path.
+ * Handles both file://path and file:///path formats.
+ */
+ @SuppressForbidden(reason = "LocalStorageProvider converts user-supplied file:// URIs to Path objects")
+ private Path toFilePath(StoragePath storagePath) {
+ String pathStr = storagePath.path();
+
+ // Handle file:// URLs - the path() method returns the path component after the scheme
+ // For file:///absolute/path, path() returns "/absolute/path"
+ // For file://relative/path, path() returns "relative/path"
+
+ if (pathStr == null || pathStr.isEmpty()) {
+ throw new IllegalArgumentException("Path cannot be empty for file:// scheme");
+ }
+
+ return PathUtils.get(pathStr);
+ }
+
+ @Override
+ public String toString() {
+ return "LocalStorageProvider{}";
+ }
+
+ private static StoragePath toStoragePath(Path filePath) {
+ return StoragePath.of(FILE_SCHEME_PREFIX + filePath.toAbsolutePath());
+ }
+
+ /**
+ * Iterator implementation for listing local directory contents.
+ */
+ private static final class LocalStorageIterator implements StorageIterator {
+ private final List entries;
+ private final Iterator iterator;
+
+ LocalStorageIterator(Path directory, boolean recursive) throws IOException {
+ this.entries = new ArrayList<>();
+
+ if (recursive) {
+ Files.walkFileTree(directory, new SimpleFileVisitor<>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+ if (attrs.isRegularFile()) {
+ StoragePath storagePath = toStoragePath(file);
+ entries.add(new StorageEntry(storagePath, attrs.size(), attrs.lastModifiedTime().toInstant()));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) {
+ // Skip entries that can't be read
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ } else {
+ try (DirectoryStream stream = Files.newDirectoryStream(directory)) {
+ for (Path entry : stream) {
+ try {
+ BasicFileAttributes attrs = Files.readAttributes(entry, BasicFileAttributes.class);
+ if (attrs.isRegularFile()) {
+ StoragePath storagePath = toStoragePath(entry);
+ entries.add(new StorageEntry(storagePath, attrs.size(), attrs.lastModifiedTime().toInstant()));
+ }
+ } catch (IOException e) {
+ // Skip entries that can't be read
+ }
+ }
+ }
+ }
+
+ this.iterator = entries.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public StorageEntry next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+ return iterator.next();
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to clean up
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/esql-datasource-http/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..9d9daa2bbcd95
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,6 @@
+ALL-UNNAMED:
+ - outbound_network
+ - files:
+ - relative_path: .
+ relative_to: shared_repo
+ mode: read
diff --git a/x-pack/plugin/esql-datasource-http/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin b/x-pack/plugin/esql-datasource-http/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
new file mode 100644
index 0000000000000..c0264edfb3b5c
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
@@ -0,0 +1 @@
+org.elasticsearch.xpack.esql.datasource.http.HttpDataSourcePlugin
diff --git a/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObjectTests.java b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObjectTests.java
new file mode 100644
index 0000000000000..37eb054d768b2
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageObjectTests.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.net.http.HttpClient;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for HttpStorageObject with Range header support.
+ *
+ * Note: These are basic unit tests that verify object creation and path handling.
+ * Full integration tests with actual HTTP requests should be done in integration test suites.
+ */
+@SuppressWarnings("unchecked")
+public class HttpStorageObjectTests extends ESTestCase {
+
+ public void testPath() {
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config);
+
+ assertEquals(path, object.path());
+ }
+
+ public void testPathWithPreKnownLength() {
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config, 12345L);
+
+ assertEquals(path, object.path());
+ }
+
+ public void testPathWithPreKnownMetadata() {
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config, 12345L, java.time.Instant.now());
+
+ assertEquals(path, object.path());
+ }
+
+ public void testInvalidRangePosition() {
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config);
+
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { object.newStream(-1, 100); });
+ assertTrue(e.getMessage().contains("position"));
+ }
+
+ public void testInvalidRangeLength() {
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config);
+
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { object.newStream(0, -1); });
+ assertTrue(e.getMessage().contains("length"));
+ }
+
+ public void testBoundedInputStreamReadsExactly() throws Exception {
+ byte[] data = "0123456789abcdefghij".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ java.io.ByteArrayInputStream source = new java.io.ByteArrayInputStream(data);
+
+ // Create a BoundedInputStream via reflection since it's private
+ HttpClient mockClient = mock(HttpClient.class);
+ StoragePath path = StoragePath.of("https://example.com/file.txt");
+ HttpConfiguration config = HttpConfiguration.defaults();
+ HttpStorageObject object = new HttpStorageObject(mockClient, path, config);
+
+ // Test that we can create the object successfully
+ assertNotNull(object);
+ assertEquals(path, object.path());
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProviderTests.java b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProviderTests.java
new file mode 100644
index 0000000000000..f5bd0936f96a7
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/HttpStorageProviderTests.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http;
+
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.time.Duration;
+import java.util.Map;
+
+/**
+ * Tests for HttpStorageProvider configuration and basic functionality.
+ * Note: Tests avoid creating real HttpClient instances to prevent thread leaks.
+ */
+public class HttpStorageProviderTests extends ESTestCase {
+
+ public void testConfigurationDefaults() {
+ HttpConfiguration config = HttpConfiguration.defaults();
+
+ assertEquals(Duration.ofSeconds(30), config.connectTimeout());
+ assertEquals(Duration.ofMinutes(5), config.requestTimeout());
+ assertTrue(config.followRedirects());
+ assertTrue(config.customHeaders().isEmpty());
+ assertEquals(3, config.maxRetries());
+ }
+
+ public void testConfigurationBuilder() {
+ HttpConfiguration config = HttpConfiguration.builder()
+ .connectTimeout(Duration.ofSeconds(15))
+ .requestTimeout(Duration.ofMinutes(3))
+ .followRedirects(false)
+ .customHeaders(Map.of("Authorization", "Bearer token"))
+ .maxRetries(2)
+ .build();
+
+ assertEquals(Duration.ofSeconds(15), config.connectTimeout());
+ assertEquals(Duration.ofMinutes(3), config.requestTimeout());
+ assertFalse(config.followRedirects());
+ assertEquals("Bearer token", config.customHeaders().get("Authorization"));
+ assertEquals(2, config.maxRetries());
+ }
+
+ public void testConfigurationBuilderValidation() {
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> { HttpConfiguration.builder().maxRetries(-1).build(); }
+ );
+ assertTrue(e.getMessage().contains("non-negative"));
+ }
+
+ public void testConfigurationBuilderNullConnectTimeout() {
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> { HttpConfiguration.builder().connectTimeout(null); }
+ );
+ assertTrue(e.getMessage().contains("connectTimeout"));
+ }
+
+ public void testConfigurationBuilderNullRequestTimeout() {
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> { HttpConfiguration.builder().requestTimeout(null); }
+ );
+ assertTrue(e.getMessage().contains("requestTimeout"));
+ }
+
+ public void testConfigurationBuilderNullCustomHeaders() {
+ IllegalArgumentException e = expectThrows(
+ IllegalArgumentException.class,
+ () -> { HttpConfiguration.builder().customHeaders(null); }
+ );
+ assertTrue(e.getMessage().contains("customHeaders"));
+ }
+
+ public void testStoragePathParsing() {
+ StoragePath path = StoragePath.of("https://example.com:8080/data/file.csv");
+
+ assertEquals("https", path.scheme());
+ assertEquals("example.com", path.host());
+ assertEquals(8080, path.port());
+ assertEquals("/data/file.csv", path.path());
+ assertEquals("file.csv", path.objectName());
+ }
+
+ public void testStoragePathWithoutPort() {
+ StoragePath path = StoragePath.of("https://example.com/data/file.csv");
+
+ assertEquals("https", path.scheme());
+ assertEquals("example.com", path.host());
+ assertEquals(-1, path.port());
+ assertEquals("/data/file.csv", path.path());
+ }
+
+ public void testListObjectsThrowsUnsupportedOperation() {
+ HttpStorageProvider provider = new HttpStorageProvider(HttpConfiguration.defaults(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
+ try {
+ StoragePath prefix = StoragePath.of("https://example.com/data/");
+ expectThrows(UnsupportedOperationException.class, () -> provider.listObjects(prefix, false));
+ expectThrows(UnsupportedOperationException.class, () -> provider.listObjects(prefix, true));
+ } finally {
+ provider.close();
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProviderTests.java b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProviderTests.java
new file mode 100644
index 0000000000000..ae1accf2bc880
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-http/src/test/java/org/elasticsearch/xpack/esql/datasource/http/local/LocalStorageProviderTests.java
@@ -0,0 +1,273 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.http.local;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.datasources.StorageEntry;
+import org.elasticsearch.xpack.esql.datasources.StorageIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests for LocalStorageProvider and LocalStorageObject.
+ */
+public class LocalStorageProviderTests extends ESTestCase {
+
+ public void testReadFullFile() throws IOException {
+ // Create a temporary file
+ Path tempFile = createTempFile("test", ".txt");
+ String content = "Hello, World!\nThis is a test file.";
+ Files.writeString(tempFile, content);
+
+ // Create storage provider and object
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath path = StoragePath.of("file://" + tempFile.toAbsolutePath());
+ StorageObject object = provider.newObject(path);
+
+ // Read the full file
+ try (
+ InputStream stream = object.newStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))
+ ) {
+ String line1 = reader.readLine();
+ String line2 = reader.readLine();
+ assertEquals("Hello, World!", line1);
+ assertEquals("This is a test file.", line2);
+ }
+ }
+
+ public void testReadRangeFromFile() throws IOException {
+ // Create a temporary file with known content
+ Path tempFile = createTempFile("test", ".txt");
+ String content = "0123456789ABCDEFGHIJ";
+ Files.writeString(tempFile, content);
+
+ // Create storage provider and object
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath path = StoragePath.of("file://" + tempFile.toAbsolutePath());
+ StorageObject object = provider.newObject(path);
+
+ // Read a range (bytes 5-9, which should be "56789")
+ try (InputStream stream = object.newStream(5, 5)) {
+ byte[] buffer = new byte[5];
+ int bytesRead = stream.read(buffer);
+ assertEquals(5, bytesRead);
+ assertEquals("56789", new String(buffer, StandardCharsets.UTF_8));
+ }
+ }
+
+ public void testFileMetadata() throws IOException {
+ // Create a temporary file
+ Path tempFile = createTempFile("test", ".txt");
+ String content = "Test content";
+ Files.writeString(tempFile, content);
+
+ // Create storage provider and object
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath path = StoragePath.of("file://" + tempFile.toAbsolutePath());
+ StorageObject object = provider.newObject(path);
+
+ // Check metadata
+ assertTrue(object.exists());
+ assertEquals(content.length(), object.length());
+ assertNotNull(object.lastModified());
+ }
+
+ public void testListDirectory() throws IOException {
+ // Create a temporary directory with some files
+ Path tempDir = createTempDir();
+ Path file1 = tempDir.resolve("file1.txt");
+ Path file2 = tempDir.resolve("file2.csv");
+ Files.writeString(file1, "content1");
+ Files.writeString(file2, "content2");
+
+ // Create storage provider
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath dirPath = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ // List directory
+ List entries = new ArrayList<>();
+ try (StorageIterator iterator = provider.listObjects(dirPath, false)) {
+ while (iterator.hasNext()) {
+ entries.add(iterator.next());
+ }
+ }
+
+ // Filter out hidden files (like .DS_Store on macOS) and ExtraFS files for the assertion
+ List fileNames = entries.stream()
+ .map(e -> e.path().objectName())
+ .filter(name -> name.startsWith(".") == false && name.startsWith("extra") == false)
+ .sorted()
+ .toList();
+ assertEquals(List.of("file1.txt", "file2.csv"), fileNames);
+ }
+
+ public void testFileNotFound() throws IOException {
+ // Use a temp directory path that doesn't exist (within allowed paths)
+ Path tempDir = createTempDir();
+ Path nonExistentFile = tempDir.resolve("nonexistent_file.txt");
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath path = StoragePath.of("file://" + nonExistentFile.toAbsolutePath());
+ StorageObject object = provider.newObject(path);
+
+ assertFalse(object.exists());
+ expectThrows(IOException.class, () -> object.newStream());
+ }
+
+ public void testSupportedSchemes() {
+ LocalStorageProvider provider = new LocalStorageProvider();
+ List schemes = provider.supportedSchemes();
+ assertEquals(1, schemes.size());
+ assertEquals("file", schemes.get(0));
+ }
+
+ public void testInvalidScheme() {
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath path = StoragePath.of("http://example.com/file.txt");
+
+ expectThrows(IllegalArgumentException.class, () -> provider.newObject(path));
+ }
+
+ // -- directory listing: non-recursive vs recursive --
+
+ public void testListDirectoryNonRecursive() throws IOException {
+ Path tempDir = createTempDir();
+ Files.createFile(tempDir.resolve("a.parquet"));
+ Files.createFile(tempDir.resolve("b.parquet"));
+ Path sub = Files.createDirectories(tempDir.resolve("sub"));
+ Files.createFile(sub.resolve("c.parquet"));
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath prefix = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ List names = collectObjectNames(provider.listObjects(prefix, false));
+ assertEquals(List.of("a.parquet", "b.parquet"), sorted(names));
+ }
+
+ public void testListDirectoryRecursive() throws IOException {
+ Path tempDir = createTempDir();
+ Files.createFile(tempDir.resolve("a.parquet"));
+ Path sub = Files.createDirectories(tempDir.resolve("sub"));
+ Files.createFile(sub.resolve("c.parquet"));
+ Path deep = Files.createDirectories(sub.resolve("deep"));
+ Files.createFile(deep.resolve("d.parquet"));
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath prefix = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ List names = collectObjectNames(provider.listObjects(prefix, true));
+ assertEquals(List.of("a.parquet", "c.parquet", "d.parquet"), sorted(names));
+ }
+
+ public void testListDirectoryRecursiveMultipleSubdirs() throws IOException {
+ Path tempDir = createTempDir();
+ for (String dir : List.of("dept_a", "dept_b", "dept_c")) {
+ Path sub = Files.createDirectories(tempDir.resolve(dir));
+ Files.createFile(sub.resolve("data.parquet"));
+ }
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath prefix = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ List entries = collectAll(provider.listObjects(prefix, true));
+ assertEquals(3, entries.size());
+ }
+
+ public void testListEmptyDirectoryReturnsNothing() throws IOException {
+ Path tempDir = createTempDir();
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath prefix = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ List entries = collectAll(provider.listObjects(prefix, true));
+ assertEquals(0, entries.size());
+ }
+
+ public void testListDirectoryRecursiveRandomTree() throws IOException {
+ Path tempDir = createTempDir();
+ String[] extensions = { ".parquet", ".csv", ".txt" };
+ int totalFiles = 0;
+
+ int dirCount = between(2, 5);
+ for (int d = 0; d < dirCount; d++) {
+ Path sub = Files.createDirectories(tempDir.resolve("dir_" + d));
+ int fileCount = between(1, 4);
+ for (int f = 0; f < fileCount; f++) {
+ String ext = extensions[random().nextInt(extensions.length)];
+ Files.createFile(sub.resolve("file_" + f + ext));
+ totalFiles++;
+ }
+ if (randomBoolean()) {
+ Path deep = Files.createDirectories(sub.resolve("nested"));
+ int deepCount = between(1, 3);
+ for (int f = 0; f < deepCount; f++) {
+ String ext = extensions[random().nextInt(extensions.length)];
+ Files.createFile(deep.resolve("deep_" + f + ext));
+ totalFiles++;
+ }
+ }
+ }
+
+ LocalStorageProvider provider = new LocalStorageProvider();
+ StoragePath prefix = StoragePath.of("file://" + tempDir.toAbsolutePath());
+
+ List entries = collectAll(provider.listObjects(prefix, true));
+ assertEquals(totalFiles, entries.size());
+
+ // Non-recursive should find zero files since all files are in subdirs
+ List flatEntries = collectAll(provider.listObjects(prefix, false));
+ assertEquals(0, flatEntries.size());
+ }
+
+ // -- helpers --
+
+ private static List collectObjectNames(StorageIterator iterator) throws IOException {
+ List names = new ArrayList<>();
+ try (iterator) {
+ while (iterator.hasNext()) {
+ String name = iterator.next().path().objectName();
+ // Filter out files created by Lucene's ExtraFS test infrastructure
+ if (name.startsWith("extra") == false) {
+ names.add(name);
+ }
+ }
+ }
+ return names;
+ }
+
+ private static List collectAll(StorageIterator iterator) throws IOException {
+ List entries = new ArrayList<>();
+ try (iterator) {
+ while (iterator.hasNext()) {
+ StorageEntry entry = iterator.next();
+ // Filter out files created by Lucene's ExtraFS test infrastructure
+ if (entry.path().objectName().startsWith("extra") == false) {
+ entries.add(entry);
+ }
+ }
+ }
+ return entries;
+ }
+
+ private static List sorted(List list) {
+ List copy = new ArrayList<>(list);
+ copy.sort(String::compareTo);
+ return copy;
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/README.md b/x-pack/plugin/esql-datasource-iceberg/README.md
new file mode 100644
index 0000000000000..22cbdc893ae70
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/README.md
@@ -0,0 +1,241 @@
+# ESQL Iceberg Data Source Plugin
+
+This plugin provides Apache Iceberg table catalog support for ESQL external data sources.
+
+## Overview
+
+The Iceberg plugin enables ESQL to query Apache Iceberg tables stored in S3. Iceberg is an open table format for large analytic datasets that provides ACID transactions, schema evolution, and efficient metadata management.
+
+## Features
+
+- **Iceberg Table Catalog** - Read Iceberg table metadata and schema
+- **Schema Discovery** - Automatically resolve schema from Iceberg metadata
+- **Partition Pruning** - Skip data files based on partition predicates
+- **Predicate Pushdown** - Push filter expressions to Iceberg for efficient scanning
+- **Arrow Vectorized Reading** - High-performance columnar data reading via Apache Arrow
+- **S3 Integration** - Native S3 file I/O for cloud-native deployments
+
+## Usage
+
+Once installed, the plugin enables querying Iceberg tables via their metadata location:
+
+```sql
+FROM "s3://my-bucket/warehouse/db/sales_table"
+| WHERE sale_date >= "2024-01-01" AND region = "EMEA"
+| STATS total = SUM(amount) BY product
+```
+
+The plugin automatically detects Iceberg tables by looking for the `metadata/` directory structure.
+
+### Iceberg Table Structure
+
+```
+s3://bucket/warehouse/db/table/
+├── data/
+│ ├── part-00000.parquet
+│ ├── part-00001.parquet
+│ └── ...
+└── metadata/
+ ├── v1.metadata.json
+ ├── v2.metadata.json
+ ├── snap-*.avro
+ └── version-hint.text
+```
+
+## Dependencies
+
+This plugin bundles significant dependencies for Iceberg, Arrow, and AWS support:
+
+### Iceberg Core
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| iceberg-core | 1.x | Iceberg table operations |
+| iceberg-aws | 1.x | S3FileIO implementation |
+| iceberg-parquet | 1.x | Parquet file support |
+| iceberg-arrow | 1.x | Arrow vectorized reading |
+
+### Apache Arrow
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| arrow-vector | 18.x | Arrow vector types |
+| arrow-memory-core | 18.x | Arrow memory management |
+| arrow-memory-unsafe | 18.x | Off-heap memory allocation |
+
+### Apache Parquet & Hadoop
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| parquet-hadoop-bundle | 1.16.0 | Parquet file reading |
+| hadoop-client-api | 3.4.1 | Hadoop Configuration |
+| hadoop-client-runtime | 3.4.1 | Hadoop runtime |
+
+### AWS SDK
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| software.amazon.awssdk:s3 | 2.x | S3 client |
+| software.amazon.awssdk:sts | 2.x | STS for role assumption |
+| software.amazon.awssdk:kms | 2.x | KMS for encryption |
+
+## Architecture
+
+```
+┌─────────────────────────────────────────┐
+│ IcebergDataSourcePlugin │
+│ implements DataSourcePlugin │
+└─────────────────┬───────────────────────┘
+ │
+ │ provides
+ ▼
+┌─────────────────────────────────────────┐
+│ IcebergTableCatalog │
+│ implements TableCatalog │
+│ │
+│ - metadata(tablePath, config) │
+│ - planScan(tablePath, config, preds) │
+│ - catalogType() → "iceberg" │
+│ - canHandle(path) │
+└─────────────────┬───────────────────────┘
+ │
+ │ uses
+ ▼
+┌─────────────────────────────────────────┐
+│ IcebergCatalogAdapter │
+│ │
+│ Adapts Iceberg's StaticTableOperations │
+│ to work with S3 metadata locations │
+└─────────────────┬───────────────────────┘
+ │
+ │ uses
+ ▼
+┌─────────────────────────────────────────┐
+│ S3FileIOFactory │
+│ │
+│ Creates S3FileIO instances for │
+│ Iceberg table operations │
+└─────────────────────────────────────────┘
+```
+
+## Supported Iceberg Features
+
+| Feature | Status |
+|---------|--------|
+| Schema discovery | Supported |
+| Column projection | Supported |
+| Partition pruning | Supported |
+| Predicate pushdown | Supported |
+| Time travel | Not yet supported |
+| Schema evolution | Read-only |
+| Hidden partitioning | Supported |
+| Row-level deletes | Not yet supported |
+
+## Supported Data Types
+
+| Iceberg Type | ESQL Type |
+|--------------|-----------|
+| boolean | BOOLEAN |
+| int | INTEGER |
+| long | LONG |
+| float | DOUBLE |
+| double | DOUBLE |
+| decimal | DOUBLE |
+| date | DATE |
+| time | TIME |
+| timestamp | DATETIME |
+| timestamptz | DATETIME |
+| string | KEYWORD |
+| uuid | KEYWORD |
+| fixed | KEYWORD |
+| binary | KEYWORD (base64) |
+| list | Not yet supported |
+| map | Not yet supported |
+| struct | Not yet supported |
+
+## Predicate Pushdown
+
+The plugin supports pushing filter predicates to Iceberg for partition pruning and data skipping:
+
+```sql
+-- Partition pruning: only scans partitions matching the predicate
+FROM "s3://bucket/table"
+| WHERE sale_date >= "2024-01-01"
+
+-- Data skipping: uses column statistics to skip row groups
+FROM "s3://bucket/table"
+| WHERE amount > 1000
+```
+
+Supported predicates:
+- Equality: `=`, `!=`
+- Comparison: `<`, `<=`, `>`, `>=`
+- NULL checks: `IS NULL`, `IS NOT NULL`
+- IN lists: `field IN (value1, value2, ...)`
+- Boolean AND/OR combinations
+
+## Configuration
+
+### S3 Configuration
+
+S3 access is configured via environment variables or Elasticsearch settings:
+
+```bash
+AWS_ACCESS_KEY_ID=your-access-key
+AWS_SECRET_ACCESS_KEY=your-secret-key
+AWS_REGION=us-east-1
+```
+
+### Iceberg-specific Settings
+
+| Setting | Default | Description |
+|---------|---------|-------------|
+| `esql.iceberg.s3.endpoint` | (AWS default) | Custom S3 endpoint (for MinIO, etc.) |
+| `esql.iceberg.s3.path_style_access` | false | Use path-style S3 access |
+
+## Building
+
+```bash
+./gradlew :x-pack:plugin:esql-datasource-iceberg:build
+```
+
+## Testing
+
+```bash
+# Unit tests
+./gradlew :x-pack:plugin:esql-datasource-iceberg:test
+
+# Integration tests (requires S3 fixture)
+./gradlew :x-pack:plugin:esql-datasource-iceberg:qa:javaRestTest
+```
+
+## Test Fixtures
+
+The `qa/` directory contains test fixtures for integration testing:
+
+```
+qa/src/javaRestTest/resources/iceberg-fixtures/
+├── employees/ # Sample Iceberg table
+│ ├── data/
+│ │ └── data.parquet
+│ └── metadata/
+│ ├── v1.metadata.json
+│ └── ...
+└── standalone/
+ └── employees.parquet # Standalone Parquet file
+```
+
+## Security Considerations
+
+- Use IAM roles for S3 access when running on AWS
+- Enable S3 bucket encryption for data at rest
+- Use VPC endpoints for private S3 access
+- Consider using AWS Lake Formation for fine-grained access control
+
+## Installation
+
+The plugin is bundled with Elasticsearch and enabled by default when the ESQL feature is available.
+
+## License
+
+Elastic License 2.0
diff --git a/x-pack/plugin/esql-datasource-iceberg/build.gradle b/x-pack/plugin/esql-datasource-iceberg/build.gradle
new file mode 100644
index 0000000000000..b50e5380e9dbf
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/build.gradle
@@ -0,0 +1,358 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.publish'
+
+esplugin {
+ name = 'esql-datasource-iceberg'
+ description = 'Iceberg table catalog support for ESQL external data sources'
+ classname = 'org.elasticsearch.xpack.esql.datasource.iceberg.IcebergDataSourcePlugin'
+ extendedPlugins = ['x-pack-esql']
+}
+
+base {
+ archivesName = 'esql-datasource-iceberg'
+}
+
+dependencies {
+ // SPI interfaces from ESQL core
+ compileOnly project(path: xpackModule('esql'))
+ compileOnly project(path: xpackModule('esql-core'))
+ compileOnly project(path: xpackModule('core'))
+ compileOnly project(':server')
+ compileOnly project(xpackModule('esql:compute'))
+
+ // Apache Iceberg with Parquet support - using parquet-hadoop-bundle to avoid jar hell from duplicate shaded classes
+ implementation("org.apache.iceberg:iceberg-core:${versions.iceberg}") {
+ exclude group: 'com.github.ben-manes.caffeine', module: 'caffeine'
+ // Exclude commons-codec to avoid jar hell - x-pack-core already provides commons-codec:1.15
+ exclude group: 'commons-codec', module: 'commons-codec'
+ // Exclude slf4j-api to avoid jar hell - x-pack-core already provides slf4j-api:2.0.6
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ // Exclude checker-qual to avoid jar hell - x-pack-esql already provides a different version
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ // Exclude Jackson to avoid jar hell - x-pack-esql already provides Jackson 2.15.0
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
+ }
+ implementation("org.apache.iceberg:iceberg-aws:${versions.iceberg}") {
+ // Exclude AWS SDK bundle - we'll declare individual modules explicitly
+ exclude group: 'software.amazon.awssdk', module: 'bundle'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ // Exclude Jackson to avoid jar hell - x-pack-esql already provides Jackson 2.15.0
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
+ }
+ implementation("org.apache.iceberg:iceberg-parquet:${versions.iceberg}") {
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop'
+ exclude group: 'org.apache.parquet', module: 'parquet-column'
+ exclude group: 'org.apache.parquet', module: 'parquet-avro'
+ exclude group: 'org.apache.parquet', module: 'parquet-format-structures'
+ exclude group: 'org.apache.parquet', module: 'parquet-common'
+ exclude group: 'org.apache.parquet', module: 'parquet-encoding'
+ exclude group: 'org.apache.parquet', module: 'parquet-jackson'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ // Exclude Jackson to avoid jar hell - x-pack-esql already provides Jackson 2.15.0
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
+ }
+ // Iceberg Arrow integration for vectorized data reading
+ implementation("org.apache.iceberg:iceberg-arrow:${versions.iceberg}") {
+ exclude group: 'org.apache.parquet', module: 'parquet-avro'
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop'
+ exclude group: 'org.apache.parquet', module: 'parquet-column'
+ exclude group: 'org.apache.parquet', module: 'parquet-format-structures'
+ exclude group: 'org.apache.parquet', module: 'parquet-common'
+ exclude group: 'org.apache.parquet', module: 'parquet-encoding'
+ exclude group: 'org.apache.parquet', module: 'parquet-jackson'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ // Exclude Jackson to avoid jar hell - x-pack-esql already provides Jackson 2.15.0
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
+ }
+ implementation('org.apache.parquet:parquet-hadoop-bundle:1.16.0')
+ implementation('com.github.ben-manes.caffeine:caffeine:2.9.3') {
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ }
+
+ // Hadoop dependencies - required at both compile time and runtime for Parquet operations.
+ //
+ // The Hadoop Configuration class is needed because:
+ // 1. ParquetFileReader has method overloads that reference Configuration in their signatures
+ // 2. ParquetReadOptions.Builder() constructor creates HadoopParquetConfiguration internally,
+ // which requires the Configuration class to be present even when using non-Hadoop code paths
+ // 3. parquet-hadoop-bundle includes shaded Parquet classes but not Hadoop Configuration
+ implementation('org.apache.hadoop:hadoop-client-api:3.4.1')
+ implementation('org.apache.hadoop:hadoop-client-runtime:3.4.1')
+
+ // Arrow dependencies (needed for Iceberg Vectorized Reader integration)
+ implementation('org.apache.arrow:arrow-vector:18.3.0')
+ implementation('org.apache.arrow:arrow-memory-core:18.3.0')
+ implementation('org.apache.arrow:arrow-memory-unsafe:18.3.0')
+
+ // Checker-qual is needed at compile time for Arrow annotations
+ // Use compileOnly to avoid jar hell at runtime - x-pack-esql already provides it
+ compileOnly 'org.checkerframework:checker-qual:3.42.0'
+
+ // AWS SDK for S3 access - following repository-s3 pattern
+ implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:apache-client:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:url-connection-client:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:auth:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-core:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-xml-protocol:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-json-protocol:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:regions:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}"
+ // KMS is required by Iceberg's AwsProperties class for encryption support
+ implementation "software.amazon.awssdk:kms:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:retries:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:s3:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:sts:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
+
+ // Apache HTTP client for AWS SDK (required by apache-client module)
+ implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+
+ runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+ runtimeOnly "commons-logging:commons-logging:${versions.commonslogging}"
+ runtimeOnly "joda-time:joda-time:2.10.14"
+ runtimeOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+ runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+ runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
+ runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+ runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}"
+ runtimeOnly "software.amazon.awssdk:arns:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:json-utils:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}"
+
+ testImplementation project(':test:framework')
+ testImplementation(testArtifact(project(xpackModule('core'))))
+ testImplementation project(xpackModule('esql'))
+ testImplementation project(xpackModule('esql-core'))
+}
+
+tasks.named("dependencyLicenses").configure {
+ mapping from: /lucene-.*/, to: 'lucene'
+ mapping from: /iceberg-.*/, to: 'iceberg'
+ mapping from: /parquet-.*/, to: 'parquet'
+ mapping from: /hadoop-.*/, to: 'hadoop'
+ mapping from: /arrow-.*/, to: 'arrow'
+ mapping from: /log4j-.*/, to: 'log4j'
+}
+
+tasks.withType(org.elasticsearch.gradle.internal.AbstractDependenciesTask).configureEach {
+ // AWS SDK module mappings
+ mapping from: 'annotations', to: 'aws-sdk-2'
+ mapping from: 'apache-client', to: 'aws-sdk-2'
+ mapping from: 'arns', to: 'aws-sdk-2'
+ mapping from: 'auth', to: 'aws-sdk-2'
+ mapping from: 'aws-core', to: 'aws-sdk-2'
+ mapping from: 'aws-json-protocol', to: 'aws-sdk-2'
+ mapping from: 'aws-query-protocol', to: 'aws-sdk-2'
+ mapping from: 'aws-xml-protocol', to: 'aws-sdk-2'
+ mapping from: 'checksums', to: 'aws-sdk-2'
+ mapping from: 'checksums-spi', to: 'aws-sdk-2'
+ mapping from: 'endpoints-spi', to: 'aws-sdk-2'
+ mapping from: 'http-auth', to: 'aws-sdk-2'
+ mapping from: 'http-auth-aws', to: 'aws-sdk-2'
+ mapping from: 'http-auth-spi', to: 'aws-sdk-2'
+ mapping from: 'http-client-spi', to: 'aws-sdk-2'
+ mapping from: 'identity-spi', to: 'aws-sdk-2'
+ mapping from: 'json-utils', to: 'aws-sdk-2'
+ mapping from: 'metrics-spi', to: 'aws-sdk-2'
+ mapping from: 'profiles', to: 'aws-sdk-2'
+ mapping from: 'protocol-core', to: 'aws-sdk-2'
+ mapping from: 'regions', to: 'aws-sdk-2'
+ mapping from: 'retries', to: 'aws-sdk-2'
+ mapping from: 'retries-spi', to: 'aws-sdk-2'
+ mapping from: 'kms', to: 'aws-sdk-2'
+ mapping from: 's3', to: 'aws-sdk-2'
+ mapping from: 'sdk-core', to: 'aws-sdk-2'
+ mapping from: 'sts', to: 'aws-sdk-2'
+ mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
+ mapping from: 'url-connection-client', to: 'aws-sdk-2'
+ mapping from: 'utils', to: 'aws-sdk-2'
+}
+
+tasks.named("thirdPartyAudit").configure {
+ ignoreMissingClasses()
+ ignoreViolations(
+ // Caffeine cache uses sun.misc.Unsafe
+ 'com.github.benmanes.caffeine.SCQHeader$HeadAndTailRef',
+ 'com.github.benmanes.caffeine.SingleConsumerQueue',
+ 'com.github.benmanes.caffeine.SingleConsumerQueue$Node',
+ 'com.github.benmanes.caffeine.base.UnsafeAccess',
+ 'com.github.benmanes.caffeine.cache.BBHeader$ReadAndWriteCounterRef',
+ 'com.github.benmanes.caffeine.cache.BBHeader$ReadCounterRef',
+ 'com.github.benmanes.caffeine.cache.BLCHeader$DrainStatusRef',
+ 'com.github.benmanes.caffeine.cache.BaseMpscLinkedArrayQueue',
+ 'com.github.benmanes.caffeine.cache.FD',
+ 'com.github.benmanes.caffeine.cache.FDA',
+ 'com.github.benmanes.caffeine.cache.FDAR',
+ 'com.github.benmanes.caffeine.cache.FDAW',
+ 'com.github.benmanes.caffeine.cache.FDAWR',
+ 'com.github.benmanes.caffeine.cache.FDR',
+ 'com.github.benmanes.caffeine.cache.FDW',
+ 'com.github.benmanes.caffeine.cache.FDWR',
+ 'com.github.benmanes.caffeine.cache.FS',
+ 'com.github.benmanes.caffeine.cache.FSA',
+ 'com.github.benmanes.caffeine.cache.FSAR',
+ 'com.github.benmanes.caffeine.cache.FSAW',
+ 'com.github.benmanes.caffeine.cache.FSAWR',
+ 'com.github.benmanes.caffeine.cache.FSR',
+ 'com.github.benmanes.caffeine.cache.FSW',
+ 'com.github.benmanes.caffeine.cache.FSWR',
+ 'com.github.benmanes.caffeine.cache.FW',
+ 'com.github.benmanes.caffeine.cache.FWA',
+ 'com.github.benmanes.caffeine.cache.FWAR',
+ 'com.github.benmanes.caffeine.cache.FWAW',
+ 'com.github.benmanes.caffeine.cache.FWAWR',
+ 'com.github.benmanes.caffeine.cache.FWR',
+ 'com.github.benmanes.caffeine.cache.FWW',
+ 'com.github.benmanes.caffeine.cache.FWWR',
+ 'com.github.benmanes.caffeine.cache.PD',
+ 'com.github.benmanes.caffeine.cache.PDA',
+ 'com.github.benmanes.caffeine.cache.PDAR',
+ 'com.github.benmanes.caffeine.cache.PDAW',
+ 'com.github.benmanes.caffeine.cache.PDAWR',
+ 'com.github.benmanes.caffeine.cache.PDR',
+ 'com.github.benmanes.caffeine.cache.PDW',
+ 'com.github.benmanes.caffeine.cache.PDWR',
+ 'com.github.benmanes.caffeine.cache.PS',
+ 'com.github.benmanes.caffeine.cache.PSA',
+ 'com.github.benmanes.caffeine.cache.PSAR',
+ 'com.github.benmanes.caffeine.cache.PSAW',
+ 'com.github.benmanes.caffeine.cache.PSAWR',
+ 'com.github.benmanes.caffeine.cache.PSR',
+ 'com.github.benmanes.caffeine.cache.PSW',
+ 'com.github.benmanes.caffeine.cache.PSWR',
+ 'com.github.benmanes.caffeine.cache.PW',
+ 'com.github.benmanes.caffeine.cache.PWA',
+ 'com.github.benmanes.caffeine.cache.PWAR',
+ 'com.github.benmanes.caffeine.cache.PWAW',
+ 'com.github.benmanes.caffeine.cache.PWAWR',
+ 'com.github.benmanes.caffeine.cache.PWR',
+ 'com.github.benmanes.caffeine.cache.PWW',
+ 'com.github.benmanes.caffeine.cache.PWWR',
+ 'com.github.benmanes.caffeine.cache.StripedBuffer',
+ 'com.github.benmanes.caffeine.cache.UnsafeAccess',
+ 'com.github.benmanes.caffeine.cache.UnsafeRefArrayAccess',
+ // Arrow memory uses sun.misc.Unsafe
+ 'org.apache.arrow.memory.util.MemoryUtil',
+ 'org.apache.arrow.memory.util.MemoryUtil$1',
+ // Hadoop internal uses sun.misc.Unsafe
+ 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
+ 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',
+ 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
+ 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
+ 'org.apache.hadoop.io.nativeio.NativeIO',
+ 'org.apache.hadoop.service.launcher.InterruptEscalator',
+ 'org.apache.hadoop.service.launcher.IrqHandler',
+ 'org.apache.hadoop.util.SignalLogger$Handler',
+ // Hadoop shaded Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ // Hadoop shaded Avro uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
+ // Hadoop shaded Curator Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ 'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor',
+ // Hadoop thirdparty Protobuf uses sun.misc.Unsafe
+ 'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
+ // Hadoop thirdparty Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ // Parquet shaded hashing uses sun.misc.Unsafe
+ 'shaded.parquet.net.openhft.hashing.HotSpotPrior7u6StringHash',
+ 'shaded.parquet.net.openhft.hashing.LongHashFunction',
+ 'shaded.parquet.net.openhft.hashing.LongTupleHashFunction',
+ 'shaded.parquet.net.openhft.hashing.ModernCompactStringHash',
+ 'shaded.parquet.net.openhft.hashing.ModernHotSpotStringHash',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess$OldUnsafeAccessBigEndian',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess$OldUnsafeAccessLittleEndian',
+ 'shaded.parquet.net.openhft.hashing.Util',
+ )
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-LICENSE.txt
new file mode 100644
index 0000000000000..7bb1330a1002b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-LICENSE.txt
@@ -0,0 +1,2261 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--------------------------------------------------------------------------------
+
+src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
+
+Some portions of this module are derived from code in the Chromium project,
+copyright (c) Google inc and (c) The Chromium Authors and licensed under the
+Apache 2.0 License or the under the 3-clause BSD license:
+
+ Copyright (c) 2013 The Chromium Authors. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from Daniel Lemire's FrameOfReference project.
+
+https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
+https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py
+
+Copyright: 2013 Daniel Lemire
+Home page: http://lemire.me/en/
+Project page: https://github.com/lemire/FrameOfReference
+License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from the TensorFlow project
+
+Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the NumPy project.
+
+https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
+
+https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
+
+Copyright (c) 2005-2017, NumPy Developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of the NumPy Developers nor the names of any
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Boost project
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the jemalloc project
+
+https://github.com/jemalloc/jemalloc
+
+Copyright (C) 2002-2017 Jason Evans .
+All rights reserved.
+Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
+Copyright (C) 2009-2017 Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice(s),
+ this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice(s),
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+--------------------------------------------------------------------------------
+
+This project includes code from the Go project, BSD 3-clause license + PATENTS
+weak patent termination clause
+(https://github.com/golang/go/blob/master/PATENTS).
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the hs2client
+
+https://github.com/cloudera/hs2client
+
+Copyright 2016 Cloudera Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+The script ci/scripts/util_wait_for_it.sh has the following license
+
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The script r/configure has the following license (MIT)
+
+Copyright (c) 2017, Jeroen Ooms and Jim Hester
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and
+cpp/src/arrow/util/logging-test.cc are adapted from
+Ray Project (https://github.com/ray-project/ray) (Apache 2.0).
+
+Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h,
+cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h,
+cpp/src/arrow/vendored/datetime/ios.mm,
+cpp/src/arrow/vendored/datetime/tz.cpp are adapted from
+Howard Hinnant's date library (https://github.com/HowardHinnant/date)
+It is licensed under MIT license.
+
+The MIT License (MIT)
+Copyright (c) 2015, 2016, 2017 Howard Hinnant
+Copyright (c) 2016 Adrian Colomitchi
+Copyright (c) 2017 Florian Dang
+Copyright (c) 2017 Paul Thompson
+Copyright (c) 2018 Tomasz Kamiński
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/util/utf8.h includes code adapted from the page
+ https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+with the following license (MIT)
+
+Copyright (c) 2008-2009 Bjoern Hoehrmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/xxhash/ have the following license
+(BSD 2-Clause License)
+
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash homepage: http://www.xxhash.com
+- xxHash source repository : https://github.com/Cyan4973/xxHash
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/double-conversion/ have the following license
+(BSD 3-Clause License)
+
+Copyright 2006-2011, the V8 project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/uriparser/ have the following license
+(BSD 3-Clause License)
+
+uriparser - RFC 3986 URI parsing library
+
+Copyright (C) 2007, Weijia Song
+Copyright (C) 2007, Sebastian Pipping
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ * Redistributions of source code must retain the above
+ copyright notice, this list of conditions and the following
+ disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials
+ provided with the distribution.
+
+ * Neither the name of the nor the names of its
+ contributors may be used to endorse or promote products
+ derived from this software without specific prior written
+ permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files under dev/tasks/conda-recipes have the following license
+
+BSD 3-clause license
+Copyright (c) 2015-2018, conda-forge
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/utfcpp/ have the following license
+
+Copyright 2006-2018 Nemanja Trifunovic
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Kudu.
+
+ * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://kudu.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Impala (incubating), formerly
+Impala. The Impala code and rights were donated to the ASF as part of the
+Incubator process after the initial code imports into Apache Parquet.
+
+Copyright: 2012 Cloudera, Inc.
+Copyright: 2016 The Apache Software Foundation.
+Home page: http://impala.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Apache Aurora.
+
+* dev/release/{release,changelog,release-candidate} are based on the scripts from
+ Apache Aurora
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://aurora.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Google styleguide.
+
+* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide.
+
+Copyright: 2009 Google Inc. All rights reserved.
+Homepage: https://github.com/google/styleguide
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This project includes code from Snappy.
+
+* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code
+ from Google's Snappy project.
+
+Copyright: 2009 Google Inc. All rights reserved.
+Homepage: https://github.com/google/snappy
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This project includes code from the manylinux project.
+
+* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py,
+ requirements.txt} are based on code from the manylinux project.
+
+Copyright: 2016 manylinux
+Homepage: https://github.com/pypa/manylinux
+License: The MIT License (MIT)
+
+--------------------------------------------------------------------------------
+
+This project includes code from the cymove project:
+
+* python/pyarrow/includes/common.pxd includes code from the cymove project
+
+The MIT License (MIT)
+Copyright (c) 2019 Omer Ozarslan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The projects includes code from the Ursabot project under the dev/archery
+directory.
+
+License: BSD 2-Clause
+
+Copyright 2019 RStudio, Inc.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project include code from mingw-w64.
+
+* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5
+
+Copyright (c) 2009 - 2013 by the mingw-w64 project
+Homepage: https://mingw-w64.org
+License: Zope Public License (ZPL) Version 2.1.
+
+---------------------------------------------------------------------------------
+
+This project include code from Google's Asylo project.
+
+* cpp/src/arrow/result.h is based on status_or.h
+
+Copyright (c) Copyright 2017 Asylo authors
+Homepage: https://asylo.dev/
+License: Apache 2.0
+
+--------------------------------------------------------------------------------
+
+This project includes code from Google's protobuf project
+
+* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
+* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h
+
+Copyright 2008 Google Inc. All rights reserved.
+Homepage: https://developers.google.com/protocol-buffers/
+License:
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it. This code is not
+standalone and requires a support library to be linked with it. This
+support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency LLVM is statically linked in certain binary distributions.
+Additionally some sections of source code have been derived from sources in LLVM
+and have been clearly labeled as such. LLVM has the following license:
+
+==============================================================================
+The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
+==============================================================================
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
+==============================================================================
+Software from third parties included in the LLVM Project:
+==============================================================================
+The LLVM Project contains third party software which is under different license
+terms. All such code will be identified clearly using at least one of two
+mechanisms:
+1) It will be in a separate directory tree with its own `LICENSE.txt` or
+ `LICENSE` file at the top containing the specific license and restrictions
+ which apply to that software, or
+2) It will contain specific license and restriction terms at the top of every
+ file.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency gRPC is statically linked in certain binary
+distributions, like the python wheels. gRPC has the following license:
+
+Copyright 2014 gRPC authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Apache Thrift is statically linked in certain binary
+distributions, like the python wheels. Apache Thrift has the following license:
+
+Apache Thrift
+Copyright (C) 2006 - 2019, The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Apache ORC is statically linked in certain binary
+distributions, like the python wheels. Apache ORC has the following license:
+
+Apache ORC
+Copyright 2013-2019 The Apache Software Foundation
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
+
+This product includes software developed by Hewlett-Packard:
+(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency zstd is statically linked in certain binary
+distributions, like the python wheels. ZSTD has the following license:
+
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+ endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency lz4 is statically linked in certain binary
+distributions, like the python wheels. lz4 has the following license:
+
+LZ4 Library
+Copyright (c) 2011-2016, Yann Collet
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency Brotli is statically linked in certain binary
+distributions, like the python wheels. Brotli has the following license:
+
+Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency rapidjson is statically linked in certain binary
+distributions, like the python wheels. rapidjson and its dependencies have the
+following licenses:
+
+Tencent is pleased to support the open source community by making RapidJSON
+available.
+
+Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
+All rights reserved.
+
+If you have downloaded a copy of the RapidJSON binary from Tencent, please note
+that the RapidJSON binary is licensed under the MIT License.
+If you have downloaded a copy of the RapidJSON source code from Tencent, please
+note that RapidJSON source code is licensed under the MIT License, except for
+the third-party components listed below which are subject to different license
+terms. Your integration of RapidJSON into your own projects may require
+compliance with the MIT License, as well as the other licenses applicable to
+the third-party components included within RapidJSON. To avoid the problematic
+JSON license in your own projects, it's sufficient to exclude the
+bin/jsonchecker/ directory, as it's the only code under the JSON license.
+A copy of the MIT License is included in this file.
+
+Other dependencies and licenses:
+
+ Open Source Software Licensed Under the BSD License:
+ --------------------------------------------------------------------
+
+ The msinttypes r29
+ Copyright (c) 2006-2013 Alexander Chemeris
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ DAMAGE.
+
+ Terms of the MIT License:
+ --------------------------------------------------------------------
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency snappy is statically linked in certain binary
+distributions, like the python wheels. snappy has the following license:
+
+Copyright 2011, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of Google Inc. nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===
+
+Some of the benchmark data in testdata/ is licensed differently:
+
+ - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
+ is licensed under the Creative Commons Attribution 3.0 license
+ (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/
+ for more information.
+
+ - kppkn.gtb is taken from the Gaviota chess tablebase set, and
+ is licensed under the MIT License. See
+ https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1
+ for more information.
+
+ - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper
+ “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA
+ Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro,
+ which is licensed under the CC-BY license. See
+ http://www.ploscompbiol.org/static/license for more ifnormation.
+
+ - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project
+ Gutenberg. The first three have expired copyrights and are in the public
+ domain; the latter does not have expired copyright, but is still in the
+ public domain according to the license information
+ (http://www.gutenberg.org/ebooks/53).
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency gflags is statically linked in certain binary
+distributions, like the python wheels. gflags has the following license:
+
+Copyright (c) 2006, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency glog is statically linked in certain binary
+distributions, like the python wheels. glog has the following license:
+
+Copyright (c) 2008, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+A function gettimeofday in utilities.cc is based on
+
+http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd
+
+The license of this code is:
+
+Copyright (c) 2003-2008, Jouni Malinen and contributors
+All Rights Reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+3. Neither the name(s) of the above-listed copyright holder(s) nor the
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency re2 is statically linked in certain binary
+distributions, like the python wheels. re2 has the following license:
+
+Copyright (c) 2009 The RE2 Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+ * Neither the name of Google Inc. nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency c-ares is statically linked in certain binary
+distributions, like the python wheels. c-ares has the following license:
+
+# c-ares license
+
+Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS
+file.
+
+Copyright 1998 by the Massachusetts Institute of Technology.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted, provided that
+the above copyright notice appear in all copies and that both that copyright
+notice and this permission notice appear in supporting documentation, and that
+the name of M.I.T. not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior permission.
+M.I.T. makes no representations about the suitability of this software for any
+purpose. It is provided "as is" without express or implied warranty.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency zlib is redistributed as a dynamically linked shared
+library in certain binary distributions, like the python wheels. In the future
+this will likely change to static linkage. zlib has the following license:
+
+zlib.h -- interface of the 'zlib' general purpose compression library
+ version 1.2.11, January 15th, 2017
+
+ Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgment in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+ Jean-loup Gailly Mark Adler
+ jloup@gzip.org madler@alumni.caltech.edu
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency openssl is redistributed as a dynamically linked shared
+library in certain binary distributions, like the python wheels. openssl
+preceding version 3 has the following license:
+
+ LICENSE ISSUES
+ ==============
+
+ The OpenSSL toolkit stays under a double license, i.e. both the conditions of
+ the OpenSSL License and the original SSLeay license apply to the toolkit.
+ See below for the actual license texts.
+
+ OpenSSL License
+ ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ * software must display the following acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ * endorse or promote products derived from this software without
+ * prior written permission. For written permission, please contact
+ * openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ * nor may "OpenSSL" appear in their names without prior written
+ * permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ * acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com). This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to. The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code. The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * "This product includes cryptographic software written by
+ * Eric Young (eay@cryptsoft.com)"
+ * The word 'cryptographic' can be left out if the rouines from the library
+ * being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ * the apps directory (application code) you must include an acknowledgement:
+ * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed. i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+
+--------------------------------------------------------------------------------
+
+This project includes code from the rtools-backports project.
+
+* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
+ from the rtools-backports project.
+
+Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
+All rights reserved.
+Homepage: https://github.com/r-windows/rtools-backports
+License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+Some code from pandas has been adapted for the pyarrow codebase. pandas is
+available under the 3-clause BSD license, which follows:
+
+pandas license
+==============
+
+Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team
+All rights reserved.
+
+Copyright (c) 2008-2011 AQR Capital Management, LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of the copyright holder nor the names of any
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+Some bits from DyND, in particular aspects of the build system, have been
+adapted from libdynd and dynd-python under the terms of the BSD 2-clause
+license
+
+The BSD 2-Clause License
+
+ Copyright (C) 2011-12, Dynamic NDArray Developers
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Dynamic NDArray Developers list:
+
+ * Mark Wiebe
+ * Continuum Analytics
+
+--------------------------------------------------------------------------------
+
+Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted
+for PyArrow. Ibis is released under the Apache License, Version 2.0.
+
+--------------------------------------------------------------------------------
+
+dev/tasks/homebrew-formulae/apache-arrow.rb has the following license:
+
+BSD 2-Clause License
+
+Copyright (c) 2009-present, Homebrew contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+
+cpp/src/arrow/vendored/base64.cpp has the following license
+
+ZLIB License
+
+Copyright (C) 2004-2017 René Nyffenegger
+
+This source code is provided 'as-is', without any express or implied
+warranty. In no event will the author be held liable for any damages arising
+from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose, including
+commercial applications, and to alter it and redistribute it freely, subject to
+the following restrictions:
+
+1. The origin of this source code must not be misrepresented; you must not
+ claim that you wrote the original source code. If you use this source code
+ in a product, an acknowledgment in the product documentation would be
+ appreciated but is not required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original source code.
+
+3. This notice may not be removed or altered from any source distribution.
+
+René Nyffenegger rene.nyffenegger@adp-gmbh.ch
+
+--------------------------------------------------------------------------------
+
+This project includes code from Folly.
+
+ * cpp/src/arrow/vendored/ProducerConsumerQueue.h
+
+is based on Folly's
+
+ * folly/Portability.h
+ * folly/lang/Align.h
+ * folly/ProducerConsumerQueue.h
+
+Copyright: Copyright (c) Facebook, Inc. and its affiliates.
+Home page: https://github.com/facebook/folly
+License: http://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+The file cpp/src/arrow/vendored/musl/strptime.c has the following license
+
+Copyright © 2005-2020 Rich Felker, et al.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file cpp/cmake_modules/BuildUtils.cmake contains code from
+
+https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49
+
+which is made available under the MIT license
+
+Copyright (c) 2019 Cristian Adam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/portable-snippets/ contain code from
+
+https://github.com/nemequ/portable-snippets
+
+and have the following copyright notice:
+
+Each source file contains a preamble explaining the license situation
+for that file, which takes priority over this file. With the
+exception of some code pulled in from other repositories (such as
+µnit, an MIT-licensed project which is used for testing), the code is
+public domain, released using the CC0 1.0 Universal dedication (*).
+
+(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/fast_float/ contain code from
+
+https://github.com/lemire/fast_float
+
+which is made available under the Apache License 2.0.
+
+--------------------------------------------------------------------------------
+
+The file python/pyarrow/vendored/docscrape.py contains code from
+
+https://github.com/numpy/numpydoc/
+
+which is made available under the BSD 2-clause license.
+
+--------------------------------------------------------------------------------
+
+The file python/pyarrow/vendored/version.py contains code from
+
+https://github.com/pypa/packaging/
+
+which is made available under both the Apache license v2.0 and the
+BSD 2-clause license.
+
+--------------------------------------------------------------------------------
+
+The files in cpp/src/arrow/vendored/pcg contain code from
+
+https://github.com/imneme/pcg-cpp
+
+and have the following copyright notice:
+
+Copyright 2014-2019 Melissa O'Neill ,
+ and the PCG Project contributors.
+
+SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+Licensed under the Apache License, Version 2.0 (provided in
+LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0)
+or under the MIT license (provided in LICENSE-MIT.txt and at
+http://opensource.org/licenses/MIT), at your option. This file may not
+be copied, modified, or distributed except according to those terms.
+
+Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either
+express or implied. See your chosen license for details.
+
+--------------------------------------------------------------------------------
+r/R/dplyr-count-tally.R (some portions)
+
+Some portions of this file are derived from code from
+
+https://github.com/tidyverse/dplyr/
+
+which is made available under the MIT license
+
+Copyright (c) 2013-2019 RStudio and others.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+The file src/arrow/util/io_util.cc contains code from the CPython project
+which is made available under the Python Software Foundation License Version 2.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency opentelemetry-cpp is statically linked in certain binary
+distributions. opentelemetry-cpp is made available under the Apache License 2.0.
+
+Copyright The OpenTelemetry Authors
+SPDX-License-Identifier: Apache-2.0
+
+--------------------------------------------------------------------------------
+
+ci/conan/ is based on code from Conan Package and Dependency Manager.
+
+Copyright (c) 2019 Conan.io
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency UCX is redistributed as a dynamically linked shared
+library in certain binary distributions. UCX has the following license:
+
+Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved.
+Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved.
+Copyright (C) 2014-2015 The University of Houston System. All rights reserved.
+Copyright (C) 2015 The University of Tennessee and The University
+ of Tennessee Research Foundation. All rights reserved.
+Copyright (C) 2016-2020 ARM Ltd. All rights reserved.
+Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved.
+Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved.
+Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved.
+Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved.
+Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved.
+Copyright (C) 2016-2020 Stony Brook University. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+The file dev/tasks/r/github.packages.yml contains code from
+
+https://github.com/ursa-labs/arrow-r-nightly
+
+which is made available under the Apache License 2.0.
+
+--------------------------------------------------------------------------------
+.github/actions/sync-nightlies/action.yml (some portions)
+
+Some portions of this file are derived from code from
+
+https://github.com/JoshPiper/rsync-docker
+
+which is made available under the MIT license
+
+Copyright (c) 2020 Joshua Piper
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+.github/actions/sync-nightlies/action.yml (some portions)
+
+Some portions of this file are derived from code from
+
+https://github.com/burnett01/rsync-deployments
+
+which is made available under the MIT license
+
+Copyright (c) 2019-2022 Contention
+Copyright (c) 2019-2022 Burnett01
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java
+java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java
+
+These file are derived from code from Netty, which is made available under the
+Apache License 2.0.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-NOTICE.txt
new file mode 100644
index 0000000000000..2089c6fb20358
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/arrow-NOTICE.txt
@@ -0,0 +1,84 @@
+Apache Arrow
+Copyright 2016-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software from the SFrame project (BSD, 3-clause).
+* Copyright (C) 2015 Dato, Inc.
+* Copyright (c) 2009 Carnegie Mellon University.
+
+This product includes software from the Feather project (Apache 2.0)
+https://github.com/wesm/feather
+
+This product includes software from the DyND project (BSD 2-clause)
+https://github.com/libdynd
+
+This product includes software from the LLVM project
+ * distributed under the University of Illinois Open Source
+
+This product includes software from the google-lint project
+ * Copyright (c) 2009 Google Inc. All rights reserved.
+
+This product includes software from the mman-win32 project
+ * Copyright https://code.google.com/p/mman-win32/
+ * Licensed under the MIT License;
+
+This product includes software from the LevelDB project
+ * Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * Moved from Kudu http://github.com/cloudera/kudu
+
+This product includes software from the CMake project
+ * Copyright 2001-2009 Kitware, Inc.
+ * Copyright 2012-2014 Continuum Analytics, Inc.
+ * All rights reserved.
+
+This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
+ * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
+
+This product includes software from the Ibis project (Apache 2.0)
+ * Copyright (c) 2015 Cloudera, Inc.
+ * https://github.com/cloudera/ibis
+
+This product includes software from Dremio (Apache 2.0)
+ * Copyright (C) 2017-2018 Dremio Corporation
+ * https://github.com/dremio/dremio-oss
+
+This product includes software from Google Guava (Apache 2.0)
+ * Copyright (C) 2007 The Guava Authors
+ * https://github.com/google/guava
+
+This product include software from CMake (BSD 3-Clause)
+ * CMake - Cross Platform Makefile Generator
+ * Copyright 2000-2019 Kitware, Inc. and Contributors
+
+The web site includes files generated by Jekyll.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache Kudu, which includes the following in
+its NOTICE file:
+
+ Apache Kudu
+ Copyright 2016 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
+ Portions of this software were developed at
+ Cloudera, Inc (http://www.cloudera.com/).
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache ORC, which includes the following in
+its NOTICE file:
+
+ Apache ORC
+ Copyright 2013-2019 The Apache Software Foundation
+
+ This product includes software developed by The Apache Software
+ Foundation (http://www.apache.org/).
+
+ This product includes software developed by Hewlett-Packard:
+ (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-LICENSE.txt
new file mode 100644
index 0000000000000..1eef70a9b9f42
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-LICENSE.txt
@@ -0,0 +1,206 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Note: Other license terms may apply to certain, identified software files contained within or distributed
+ with the accompanying software if such terms are included in the directory containing the accompanying software.
+ Such other license terms will then apply in lieu of the terms of the software license above.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-NOTICE.txt
new file mode 100644
index 0000000000000..f3c4db7d1724e
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/aws-sdk-2-NOTICE.txt
@@ -0,0 +1,26 @@
+AWS SDK for Java 2.0
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+This product includes software developed by
+Amazon Technologies, Inc (http://www.amazon.com/).
+
+**********************
+THIRD PARTY COMPONENTS
+**********************
+This software includes third party software subject to the following copyrights:
+- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+- Apache Commons Lang - https://github.com/apache/commons-lang
+- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams
+- Jackson-core - https://github.com/FasterXML/jackson-core
+- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
+
+The licenses for these third party components are included in LICENSE.txt
+
+- For Apache Commons Lang see also this required NOTICE:
+ Apache Commons Lang
+ Copyright 2001-2020 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (https://www.apache.org/).
+
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-LICENSE.txt
new file mode 100644
index 0000000000000..325535ee15ed5
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Support. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-NOTICE.txt
new file mode 100644
index 0000000000000..5cf47edbf236b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/caffeine-NOTICE.txt
@@ -0,0 +1,2 @@
+Caffeine (High performance caching library)
+Copyright Ben Manes. All Rights Reserved.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-NOTICE.txt
new file mode 100644
index 0000000000000..62fc5816c996b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/hadoop-NOTICE.txt
@@ -0,0 +1,2 @@
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-LICENSE.txt
new file mode 100644
index 0000000000000..325535ee15ed5
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Support. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-NOTICE.txt
new file mode 100644
index 0000000000000..b1dc399877bd3
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/iceberg-NOTICE.txt
@@ -0,0 +1,25 @@
+Apache Iceberg
+Copyright 2017-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains code from the following projects:
+
+Apache Avro (https://avro.apache.org/)
+* Copyright 2010-2019 The Apache Software Foundation
+* License: Apache License 2.0
+
+Apache ORC (https://orc.apache.org/)
+* Copyright 2013-2019 The Apache Software Foundation
+* License: Apache License 2.0
+
+Apache Parquet (https://parquet.apache.org/)
+* Copyright 2012-2019 The Apache Software Foundation
+* License: Apache License 2.0
+
+Google Guava (https://github.com/google/guava)
+* Copyright (C) 2007 The Guava Authors
+* License: Apache License 2.0
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-NOTICE.txt
new file mode 100644
index 0000000000000..dffbcf31cacf6
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/joda-time-NOTICE.txt
@@ -0,0 +1,5 @@
+=============================================================================
+= NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
+=============================================================================
+This product includes software developed by
+Joda.org (http://www.joda.org/).
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-LICENSE.txt
new file mode 100644
index 0000000000000..f57fe7c0213a9
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-LICENSE.txt
@@ -0,0 +1,201 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, ticesnames, and attributions from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Support. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-NOTICE.txt
new file mode 100644
index 0000000000000..63f78a662db1b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/parquet-NOTICE.txt
@@ -0,0 +1,13 @@
+Apache Parquet
+Copyright 2014-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project includes code from https://github.com/lemire/JavaFastPFOR
+Copyright 2013 Daniel Lemire and Owen Kaser
+Apache License Version 2.0
+
+This project includes code from https://github.com/lemire/streamvbyte
+Copyright 2017 Daniel Lemire
+Apache License Version 2.0
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/reactive-streams-LICENSE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/reactive-streams-LICENSE.txt
new file mode 100644
index 0000000000000..1e141c13ddba2
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/licenses/reactive-streams-LICENSE.txt
@@ -0,0 +1,7 @@
+MIT No Attribution
+
+Copyright 2014 Reactive Streams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/x-pack/plugin/esql-datasource-iceberg/licenses/reactive-streams-NOTICE.txt b/x-pack/plugin/esql-datasource-iceberg/licenses/reactive-streams-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/build.gradle b/x-pack/plugin/esql-datasource-iceberg/qa/build.gradle
new file mode 100644
index 0000000000000..8f8d54236971d
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/build.gradle
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
+
+dependencies {
+ // Test fixtures and spec reader infrastructure from ESQL
+ javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
+ javaRestTestImplementation project(xpackModule('esql:qa:server'))
+ javaRestTestImplementation project(xpackModule('esql'))
+ javaRestTestImplementation(project(path: xpackModule('esql'), configuration: 'testRuntimeElements'))
+
+ // S3 fixture infrastructure for mocking S3 operations
+ javaRestTestImplementation project(':test:fixtures:s3-fixture')
+ javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
+
+ // Apache Iceberg with Parquet support - use same versions as parent module
+ javaRestTestImplementation("org.apache.iceberg:iceberg-core:${versions.iceberg}") {
+ exclude group: 'com.github.ben-manes.caffeine', module: 'caffeine'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ }
+ javaRestTestImplementation("org.apache.iceberg:iceberg-aws:${versions.iceberg}") {
+ exclude group: 'software.amazon.awssdk', module: 'bundle'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ }
+ javaRestTestImplementation("org.apache.iceberg:iceberg-parquet:${versions.iceberg}") {
+ exclude group: 'org.apache.parquet', module: 'parquet-hadoop'
+ exclude group: 'org.apache.parquet', module: 'parquet-column'
+ exclude group: 'org.apache.parquet', module: 'parquet-avro'
+ exclude group: 'org.apache.parquet', module: 'parquet-format-structures'
+ exclude group: 'org.apache.parquet', module: 'parquet-common'
+ exclude group: 'org.apache.parquet', module: 'parquet-encoding'
+ exclude group: 'org.apache.parquet', module: 'parquet-jackson'
+ exclude group: 'commons-codec', module: 'commons-codec'
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ }
+ javaRestTestImplementation('org.apache.parquet:parquet-hadoop-bundle:1.16.0')
+ javaRestTestImplementation('com.github.ben-manes.caffeine:caffeine:2.9.3') {
+ exclude group: 'org.checkerframework', module: 'checker-qual'
+ }
+
+ // Repository S3 module for cluster
+ clusterModules project(':modules:repository-s3')
+ clusterPlugins project(':plugins:mapper-size')
+ clusterPlugins project(':plugins:mapper-murmur3')
+
+ // The Iceberg datasource plugin under test
+ clusterPlugins project(xpackModule('esql-datasource-iceberg'))
+ clusterPlugins project(xpackModule('esql-datasource-s3'))
+}
+
+// Test resources (iceberg-fixtures) are now local to this module
+// in src/javaRestTest/resources/
+
+// InteractiveFixtureManual is intentionally not named with an IT suffix to prevent automatic execution;
+// it is a manual interactive testing tool, not a regular integration test.
+tasks.named('javaRestTestTestingConventions').configure {
+ baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+ suffix 'IT'
+ suffix 'Manual'
+}
+
+tasks.named("forbiddenPatterns").configure {
+ exclude '**/*.parquet'
+ exclude '**/*.avro'
+ exclude '**/.*.crc'
+}
+
+tasks.named('javaRestTest') {
+ usesDefaultDistribution("to be triaged")
+ maxParallelForks = 1
+
+ // Increase timeouts for S3/Iceberg operations which may take longer than standard queries
+ systemProperty 'tests.rest.client_timeout', '60'
+ systemProperty 'tests.rest.socket_timeout', '60'
+
+ // Enable more verbose logging for debugging
+ testLogging {
+ events = ["passed", "skipped", "failed"]
+ exceptionFormat = "full"
+ showStandardStreams = false
+ }
+}
+
+restResources {
+ restApi {
+ include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'cluster', 'capabilities', 'index'
+ }
+ restTests {
+ includeXpack 'esql'
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/Clusters.java b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/Clusters.java
new file mode 100644
index 0000000000000..e145693b2cfbb
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/Clusters.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.iceberg;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.ACCESS_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.SECRET_KEY;
+
+/**
+ * Cluster configuration for Iceberg integration tests.
+ * Provides ES cluster setup with S3 repository plugin and Iceberg catalog configuration.
+ */
+public class Clusters {
+
+ /**
+ * Creates a test cluster configured for Iceberg integration testing.
+ *
+ * @param s3EndpointSupplier supplier for the S3 fixture endpoint URL
+ * @param configProvider additional cluster configuration provider
+ * @return configured ElasticsearchCluster
+ */
+ public static ElasticsearchCluster testCluster(Supplier s3EndpointSupplier, LocalClusterConfigProvider configProvider) {
+ return ElasticsearchCluster.local()
+ .distribution(DistributionType.DEFAULT)
+ .shared(true)
+ // Enable S3 repository plugin for S3 access
+ .module("repository-s3")
+ // Basic cluster settings
+ .setting("xpack.security.enabled", "false")
+ .setting("xpack.license.self_generated.type", "trial")
+ // Disable ML to avoid native code loading issues in some environments
+ .setting("xpack.ml.enabled", "false")
+ // S3 client configuration for accessing the S3HttpFixture
+ .setting("s3.client.default.endpoint", s3EndpointSupplier)
+ // S3 credentials must be stored in keystore, not as regular settings
+ .keystore("s3.client.default.access_key", ACCESS_KEY)
+ .keystore("s3.client.default.secret_key", SECRET_KEY)
+ // Disable SSL for HTTP fixture
+ .setting("s3.client.default.protocol", "http")
+ // Disable AWS SDK profile file loading by pointing to non-existent files
+ // This prevents the SDK from trying to read ~/.aws/credentials and ~/.aws/config
+ // which would violate Elasticsearch entitlements
+ .environment("AWS_CONFIG_FILE", "/dev/null/aws/config")
+ .environment("AWS_SHARED_CREDENTIALS_FILE", "/dev/null/aws/credentials")
+ // Arrow's unsafe memory allocator requires access to java.nio internals
+ .jvmArg("--add-opens=java.base/java.nio=ALL-UNNAMED")
+ // Configure Arrow to use unsafe memory allocator instead of netty
+ // This must be set as a JVM arg to take effect before any Arrow classes are loaded
+ .jvmArg("-Darrow.allocation.manager.type=Unsafe")
+ // Apply any additional configuration
+ .apply(() -> configProvider)
+ .build();
+ }
+
+ /**
+ * Creates a test cluster with default configuration.
+ *
+ * @param s3EndpointSupplier supplier for the S3 fixture endpoint URL
+ * @return configured ElasticsearchCluster
+ */
+ public static ElasticsearchCluster testCluster(Supplier s3EndpointSupplier) {
+ return testCluster(s3EndpointSupplier, config -> {});
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecIT.java b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecIT.java
new file mode 100644
index 0000000000000..3554020b3f511
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecIT.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.iceberg;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.SpecReader;
+import org.junit.ClassRule;
+
+import java.net.URL;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
+import static org.junit.Assert.assertTrue;
+
+/** Integration tests for Iceberg tables with metadata (loads iceberg-*.csv-spec). */
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+@AwaitsFix(bugUrl = "Iceberg integration tests disabled pending stabilization")
+public class IcebergSpecIT extends IcebergSpecTestCase {
+
+ /** Elasticsearch cluster with S3 fixture and Iceberg catalog for testing. */
+ @ClassRule
+ public static ElasticsearchCluster cluster = Clusters.testCluster(() -> s3Fixture.getAddress());
+
+ public IcebergSpecIT(
+ String fileName,
+ String groupName,
+ String testName,
+ Integer lineNumber,
+ CsvTestCase testCase,
+ String instructions
+ ) {
+ super(fileName, groupName, testName, lineNumber, testCase, instructions);
+ }
+
+ @Override
+ protected String getTestRestCluster() {
+ return cluster.getHttpAddresses();
+ }
+
+ @ParametersFactory(argumentFormatting = "csv-spec:%2$s.%3$s")
+ public static List readScriptSpec() throws Exception {
+ List urls = classpathResources("/iceberg-*.csv-spec");
+ assertTrue("No iceberg-*.csv-spec files found", urls.size() > 0);
+ return SpecReader.readScriptSpec(urls, specParser());
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecTestCase.java b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecTestCase.java
new file mode 100644
index 0000000000000..8d3126a482f7a
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/IcebergSpecTestCase.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.qa.iceberg;
+
+import org.apache.iceberg.aws.s3.S3FileIO;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils;
+import org.elasticsearch.xpack.esql.qa.rest.AbstractExternalSourceSpecTestCase;
+import org.junit.BeforeClass;
+
+/**
+ * Base test class for Iceberg integration tests using S3HttpFixture.
+ * Extends {@link AbstractExternalSourceSpecTestCase} with Iceberg-specific functionality.
+ *
+ * Iceberg tests always use S3 storage backend since Iceberg requires metadata files.
+ * The format is "iceberg" to indicate Iceberg table format (not standalone parquet).
+ */
+public abstract class IcebergSpecTestCase extends AbstractExternalSourceSpecTestCase {
+
+ private static final Logger logger = LogManager.getLogger(IcebergSpecTestCase.class);
+
+ /**
+ * Verify that Iceberg fixtures were loaded successfully.
+ */
+ @BeforeClass
+ public static void verifyIcebergFixturesLoaded() {
+ logger.info("=== Verifying Iceberg Fixtures ===");
+
+ try {
+ var logs = getRequestLogs();
+ logger.info("Total fixture operations logged: {}", logs.size());
+
+ boolean hasEmployeesMetadata = logs.stream()
+ .anyMatch(log -> log.getPath() != null && log.getPath().contains("employees/metadata"));
+
+ boolean hasEmployeesParquet = logs.stream()
+ .anyMatch(log -> log.getPath() != null && log.getPath().contains("standalone/employees.parquet"));
+
+ if (hasEmployeesMetadata) {
+ logger.info("✓ employees Iceberg table metadata found - using Iceberg format");
+ } else if (hasEmployeesParquet) {
+ logger.info("✓ standalone/employees.parquet found - using legacy Parquet format");
+ } else {
+ logger.warn("✗ employees fixture NOT found - tests may fail");
+ }
+
+ long parquetFiles = logs.stream().filter(log -> log.getPath() != null && log.getPath().endsWith(".parquet")).count();
+ long metadataFiles = logs.stream().filter(log -> log.getPath() != null && log.getPath().contains("metadata")).count();
+
+ logger.info("Fixture summary: {} Parquet files, {} metadata files", parquetFiles, metadataFiles);
+
+ } catch (Exception e) {
+ logger.error("Failed to verify fixtures", e);
+ }
+
+ logger.info("=== Iceberg Setup Verification Complete ===");
+ }
+
+ protected IcebergSpecTestCase(
+ String fileName,
+ String groupName,
+ String testName,
+ Integer lineNumber,
+ CsvTestCase testCase,
+ String instructions
+ ) {
+ // Iceberg tests use S3 storage backend and "iceberg" format (no template transformation needed)
+ super(fileName, groupName, testName, lineNumber, testCase, instructions, StorageBackend.S3, "iceberg");
+ }
+
+ /**
+ * Verifies that Iceberg metadata files were accessed during test execution.
+ */
+ protected void verifyIcebergMetadataUsed() {
+ var logs = getRequestLogs();
+
+ boolean accessedMetadataJson = logs.stream().anyMatch(log -> log.getPath() != null && log.getPath().contains("metadata.json"));
+
+ boolean accessedManifestList = logs.stream().anyMatch(log -> log.getPath() != null && log.getPath().contains("/metadata/snap-"));
+
+ boolean accessedManifest = logs.stream().anyMatch(log -> log.getPath() != null && log.getPath().matches(".*metadata/.*\\.avro"));
+
+ logger.info("Iceberg metadata usage verification:");
+ logger.info(" - Metadata JSON accessed: {}", accessedMetadataJson);
+ logger.info(" - Manifest list accessed: {}", accessedManifestList);
+ logger.info(" - Manifest file accessed: {}", accessedManifest);
+
+ if (accessedMetadataJson || accessedManifestList || accessedManifest) {
+ logger.info("✓ Confirmed using Iceberg table format");
+ } else {
+ logger.warn("✗ No Iceberg metadata files accessed - may be using standalone Parquet format");
+ }
+ }
+
+ /**
+ * Returns true if Iceberg metadata was used in the current test.
+ */
+ protected boolean wasIcebergMetadataUsed() {
+ var logs = getRequestLogs();
+ return logs.stream()
+ .anyMatch(
+ log -> log.getPath() != null
+ && (log.getPath().contains("metadata.json")
+ || log.getPath().contains("/metadata/snap-")
+ || log.getPath().matches(".*metadata/.*\\.avro"))
+ );
+ }
+
+ /**
+ * Creates an S3FileIO configured to use the S3HttpFixture.
+ */
+ protected static S3FileIO createS3FileIO() {
+ return S3FixtureUtils.createS3FileIO(s3Fixture.getAddress());
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/InteractiveFixtureManual.java b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/InteractiveFixtureManual.java
new file mode 100644
index 0000000000000..ca81f6ce93c9d
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/InteractiveFixtureManual.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.iceberg;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.DataSourcesS3HttpFixture;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.S3RequestLog;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.core.Booleans.parseBoolean;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.ACCESS_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.BUCKET;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.SECRET_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.WAREHOUSE;
+
+/**
+ * Interactive fixture runner for manual testing of ESQL External command with Parquet/S3.
+ *
+ * IMPORTANT: This class is named "Manual" (not "IT" or "Test") to prevent automatic
+ * execution during regular builds. It must be explicitly selected to run.
+ *
+ * This starts:
+ *
+ * S3HttpFixture on port 9345 serving Parquet files from src/test/resources/iceberg-fixtures/
+ * Elasticsearch cluster on port 9200 configured to access the fixture via S3
+ *
+ *
+ * Then waits indefinitely (or for configured time) to allow manual queries via curl,
+ * Kibana Dev Console, or other tools.
+ *
+ * Usage:
+ *
+ * # Explicit test selection (required):
+ * ./gradlew :x-pack:plugin:esql:qa:server:iceberg:javaRestTest \
+ * --tests "*InteractiveFixtureManual*"
+ *
+ *
+ * Optional System Properties:
+ *
+ * {@code -Dtests.fixture.wait_minutes=N} - Wait N minutes (0 = indefinite, default: 0)
+ * {@code -Dtests.fixture.show_blobs=true} - List all loaded fixtures (default: false)
+ * {@code -Dtests.fixture.show_logs=false} - Show S3 request logs (default: true)
+ *
+ *
+ * Fixed Ports:
+ *
+ * Elasticsearch: http://localhost:9200
+ * S3/HTTP Fixture: http://localhost:9345
+ *
+ * Press Ctrl+C to stop when running indefinitely.
+ */
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+@TimeoutSuite(millis = 7 * 24 * 60 * 60 * 1000) // 7 days - effectively no timeout
+@AwaitsFix(bugUrl = "Iceberg integration tests disabled pending stabilization")
+public class InteractiveFixtureManual extends ESRestTestCase {
+
+ /** Fixed port for Elasticsearch */
+ private static final int ES_PORT = 9200;
+
+ /** Fixed port for S3/HTTP fixture */
+ private static final int S3_FIXTURE_PORT = 9345;
+
+ private static final PrintStream out = stderr();
+
+ /** S3 HTTP fixture serving test data on fixed port */
+ public static DataSourcesS3HttpFixture s3Fixture = new DataSourcesS3HttpFixture(S3_FIXTURE_PORT);
+
+ /** Elasticsearch cluster with S3 fixture for interactive testing on fixed port */
+ public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+ .distribution(DistributionType.DEFAULT)
+ // Fixed port for easy access
+ .setting("http.port", String.valueOf(ES_PORT))
+ // Enable S3 repository plugin for S3 access
+ .module("repository-s3")
+ // Basic cluster settings
+ .setting("xpack.security.enabled", "false")
+ .setting("xpack.license.self_generated.type", "trial")
+ // Disable ML to avoid native code loading issues in some environments
+ .setting("xpack.ml.enabled", "false")
+ // S3 client configuration for accessing the S3HttpFixture
+ .setting("s3.client.default.endpoint", () -> s3Fixture.getAddress())
+ // S3 credentials must be stored in keystore, not as regular settings
+ .keystore("s3.client.default.access_key", ACCESS_KEY)
+ .keystore("s3.client.default.secret_key", SECRET_KEY)
+ // Disable SSL for HTTP fixture
+ .setting("s3.client.default.protocol", "http")
+ // Disable AWS SDK profile file loading
+ .environment("AWS_CONFIG_FILE", "/dev/null/aws/config")
+ .environment("AWS_SHARED_CREDENTIALS_FILE", "/dev/null/aws/credentials")
+ // Arrow's unsafe memory allocator requires access to java.nio internals
+ .jvmArg("--add-opens=java.base/java.nio=ALL-UNNAMED")
+ .jvmArg("-Darrow.allocation.manager.type=Unsafe")
+ .build();
+
+ /** Rule chain ensures s3Fixture starts before cluster (cluster depends on s3Fixture address) */
+ @ClassRule
+ public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+ // Wait time in minutes (configurable via system property, 0 = indefinite)
+ private static final int WAIT_MINUTES = Integer.parseInt(System.getProperty("tests.fixture.wait_minutes", "0"));
+
+ // Whether to show all loaded fixtures
+ private static final boolean SHOW_BLOBS = parseBoolean(System.getProperty("tests.fixture.show_blobs", "false"));
+
+ // Whether to show S3 request logs during interactive session
+ private static final boolean SHOW_LOGS = parseBoolean(System.getProperty("tests.fixture.show_logs", "true"));
+
+ // Message templates for output
+ private MessageTemplates messages;
+
+ @BeforeClass
+ public static void loadFixtures() {
+ s3Fixture.loadFixturesFromResources();
+ }
+
+ @Override
+ protected String getTestRestCluster() {
+ return cluster.getHttpAddresses();
+ }
+
+ /**
+ * Main interactive entry point that starts the fixture and cluster, then waits.
+ * This is a "test" only in name - it doesn't assert anything, just keeps the fixture running.
+ */
+ public void testInteractiveMode() throws Exception {
+ // Load message templates
+ loadMessages();
+
+ // Display information
+ messages.print("banner");
+ printClusterInfo();
+ printFixtureInfo();
+ printAvailableFixtures();
+ messages.print("example_queries");
+ printWaitMessage();
+
+ // Wait for the specified duration
+ waitWithProgress(WAIT_MINUTES);
+
+ if (SHOW_LOGS) {
+ printRequestLogs();
+ }
+
+ messages.print("shutdown");
+ }
+
+ private void loadMessages() throws Exception {
+ messages = MessageTemplates.load("/interactive-fixture-messages.txt");
+
+ // Set common variables
+ String fixtureUrl = s3Fixture.getAddress();
+ messages.set("es_url", cluster.getHttpAddresses())
+ .set("s3_endpoint", fixtureUrl)
+ .set("fixture_url", fixtureUrl)
+ .set("bucket", BUCKET)
+ .set("warehouse", WAREHOUSE)
+ .set("access_key", ACCESS_KEY)
+ .set("secret_key", SECRET_KEY);
+
+ // Extract port from URL
+ try {
+ java.net.URI uri = new java.net.URI(fixtureUrl);
+ int port = uri.getPort();
+ messages.set("port", port > 0 ? String.valueOf(port) : "default");
+ } catch (Exception e) {
+ messages.set("port", "(unable to parse)");
+ }
+ }
+
+ private void printClusterInfo() {
+ messages.print("cluster_info");
+ }
+
+ private void printFixtureInfo() {
+ messages.print("fixture_info");
+ }
+
+ private void printAvailableFixtures() {
+ var handler = s3Fixture.getHandler();
+ var blobs = handler.blobs();
+
+ // Count fixtures by type
+ long parquetCount = blobs.keySet().stream().filter(key -> key.endsWith(".parquet")).count();
+ long metadataCount = blobs.keySet().stream().filter(key -> key.contains("metadata")).count();
+ long otherCount = blobs.size() - parquetCount - metadataCount;
+
+ messages.set("total_files", blobs.size())
+ .set("parquet_count", parquetCount)
+ .set("metadata_count", metadataCount)
+ .set("other_count", otherCount > 0 ? String.valueOf(otherCount) : "");
+
+ messages.print("fixtures_header");
+
+ if (SHOW_BLOBS) {
+ messages.print("fixtures_show_all");
+ blobs.keySet().stream().sorted().forEach(key -> {
+ long size = blobs.get(key).length();
+ out.printf(Locale.ROOT, " %-80s %10s%n", key, MessageTemplates.formatBytes(size));
+ });
+ } else {
+ messages.print("fixtures_show_key");
+ blobs.keySet().stream().filter(key -> key.contains("employees") || key.contains("standalone")).sorted().forEach(key -> {
+ long size = blobs.get(key).length();
+ out.printf(Locale.ROOT, " %-80s %10s%n", key, MessageTemplates.formatBytes(size));
+ });
+ messages.print("fixtures_footer");
+ }
+ }
+
+ private void printWaitMessage() {
+ if (WAIT_MINUTES == 0) {
+ messages.print("wait_indefinite");
+ } else {
+ messages.set("wait_minutes", WAIT_MINUTES);
+ messages.print("wait_timed");
+ }
+ }
+
+ private void waitWithProgress(int minutes) throws InterruptedException {
+ long intervalMillis = 60L * 1000L; // Update every minute
+
+ if (minutes == 0) {
+ // Run indefinitely
+ long startTime = System.currentTimeMillis();
+ while (true) {
+ Thread.sleep(intervalMillis);
+ long elapsedMillis = System.currentTimeMillis() - startTime;
+ long elapsedMinutes = elapsedMillis / (60L * 1000L);
+ long elapsedSeconds = (elapsedMillis % (60L * 1000L)) / 1000L;
+
+ messages.set("elapsed_time", MessageTemplates.formatTime(elapsedMinutes, elapsedSeconds));
+ messages.print("progress_indefinite");
+ }
+ } else {
+ // Run for specified time
+ long totalMillis = minutes * 60L * 1000L;
+ long elapsedMillis = 0;
+ long startTime = System.currentTimeMillis();
+
+ while (elapsedMillis < totalMillis) {
+ Thread.sleep(intervalMillis);
+ elapsedMillis = System.currentTimeMillis() - startTime;
+
+ long remainingMillis = totalMillis - elapsedMillis;
+ long remainingMinutes = remainingMillis / (60L * 1000L);
+ long remainingSeconds = (remainingMillis % (60L * 1000L)) / 1000L;
+
+ messages.set("remaining_time", MessageTemplates.formatTime(remainingMinutes, remainingSeconds));
+ messages.print("progress_timed");
+ }
+ }
+ }
+
+ private void printRequestLogs() {
+ out.println();
+ out.println("--------------------------------------------------------------------------------");
+ out.println("S3 REQUEST LOG SUMMARY");
+ out.println("--------------------------------------------------------------------------------");
+
+ List logs = S3FixtureUtils.getRequestLogs();
+
+ if (logs.isEmpty()) {
+ out.println(" No S3 requests were made during this session.");
+ return;
+ }
+
+ out.println(" Total requests: " + logs.size());
+ out.println();
+ out.println(" Requests by type:");
+
+ Map byType = logs.stream().collect(Collectors.groupingBy(S3RequestLog::getRequestType, Collectors.counting()));
+
+ byType.entrySet()
+ .stream()
+ .sorted(Map.Entry.comparingByValue().reversed())
+ .forEach(entry -> out.printf(Locale.ROOT, " %-25s %5d%n", entry.getKey(), entry.getValue()));
+
+ out.println();
+ out.println(" Unique paths accessed:");
+ logs.stream().map(S3RequestLog::getPath).distinct().sorted().limit(20).forEach(path -> out.printf(Locale.ROOT, " %s%n", path));
+
+ if (logs.stream().map(S3RequestLog::getPath).distinct().count() > 20) {
+ out.println(" ... (showing first 20 paths)");
+ }
+ }
+
+ @SuppressForbidden(reason = "System.err is intentional for this interactive manual testing tool")
+ private static PrintStream stderr() {
+ return System.err;
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/MessageTemplates.java b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/MessageTemplates.java
new file mode 100644
index 0000000000000..cacb015c88008
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/iceberg/MessageTemplates.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.iceberg;
+
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Simple message template engine for loading and rendering messages from a template file.
+ * Supports variable substitution using {{variable_name}} syntax and conditional blocks.
+ *
+ * Output goes to a logger at WARN level to ensure visibility in test output.
+ */
+public class MessageTemplates {
+
+ private static final Logger logger = LogManager.getLogger(MessageTemplates.class);
+
+ private final Map templates = new HashMap<>();
+ private final Map variables = new HashMap<>();
+ private final PrintStream out;
+
+ /**
+ * Load templates from a resource file.
+ * Uses System.err for output to ensure visibility (bypasses test output capture).
+ *
+ * @param resourcePath path to the template file
+ * @return MessageTemplates instance
+ * @throws IOException if the file cannot be read
+ */
+ public static MessageTemplates load(String resourcePath) throws IOException {
+ MessageTemplates templates = new MessageTemplates(stderr());
+ templates.loadFromResource(resourcePath);
+ return templates;
+ }
+
+ /**
+ * Create a MessageTemplates instance with custom output stream.
+ *
+ * @param out the output stream to use for printing
+ */
+ public MessageTemplates(PrintStream out) {
+ this.out = out;
+ }
+
+ /**
+ * Create a MessageTemplates instance using System.err.
+ */
+ public MessageTemplates() {
+ this(stderr());
+ }
+
+ /**
+ * Set a variable value for template substitution.
+ *
+ * @param name variable name
+ * @param value variable value
+ * @return this instance for chaining
+ */
+ public MessageTemplates set(String name, String value) {
+ variables.put(name, value);
+ return this;
+ }
+
+ /**
+ * Set a variable value for template substitution.
+ *
+ * @param name variable name
+ * @param value variable value (converted to string)
+ * @return this instance for chaining
+ */
+ public MessageTemplates set(String name, long value) {
+ return set(name, String.valueOf(value));
+ }
+
+ /**
+ * Set a variable value for template substitution.
+ *
+ * @param name variable name
+ * @param value variable value (converted to string)
+ * @return this instance for chaining
+ */
+ public MessageTemplates set(String name, int value) {
+ return set(name, String.valueOf(value));
+ }
+
+ /**
+ * Get a rendered template by name.
+ *
+ * @param name template name (from [section] in the file)
+ * @return rendered template with variables substituted
+ */
+ public String get(String name) {
+ String template = templates.get(name);
+ if (template == null) {
+ return "[Template not found: " + name + "]";
+ }
+ return render(template);
+ }
+
+ /**
+ * Print a template to the output stream.
+ *
+ * @param name template name
+ */
+ public void print(String name) {
+ out.println(get(name));
+ }
+
+ /**
+ * Print a formatted string to the output stream.
+ *
+ * @param format format string
+ * @param args format arguments
+ */
+ public void printf(String format, Object... args) {
+ out.printf(Locale.ROOT, format, args);
+ }
+
+ /**
+ * Print a newline.
+ */
+ public void println() {
+ out.println();
+ }
+
+ private void loadFromResource(String resourcePath) throws IOException {
+ InputStream is = getClass().getResourceAsStream(resourcePath);
+ if (is == null) {
+ throw new IOException("Resource not found: " + resourcePath);
+ }
+
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
+ String currentSection = null;
+ StringBuilder content = new StringBuilder();
+
+ String line;
+ while ((line = reader.readLine()) != null) {
+ // Skip comments
+ if (line.trim().startsWith("#")) {
+ continue;
+ }
+
+ // Check for section header [name]
+ if (line.startsWith("[") && line.endsWith("]")) {
+ // Save previous section
+ if (currentSection != null) {
+ templates.put(currentSection, content.toString());
+ }
+
+ // Start new section
+ currentSection = line.substring(1, line.length() - 1);
+ content = new StringBuilder();
+ } else if (currentSection != null) {
+ // Append to current section
+ content.append(line).append("\n");
+ }
+ }
+
+ // Save last section
+ if (currentSection != null) {
+ templates.put(currentSection, content.toString());
+ }
+ }
+ }
+
+ private String render(String template) {
+ String result = template;
+
+ // Handle conditional blocks: {{#var}}content{{/var}}
+ // Shows content only if variable exists and is not empty
+ Pattern conditionalPattern = Pattern.compile("\\{\\{#(\\w+)\\}\\}([^{]*)\\{\\{/\\1\\}\\}");
+ Matcher matcher = conditionalPattern.matcher(result);
+ StringBuffer sb = new StringBuffer();
+ while (matcher.find()) {
+ String varName = matcher.group(1);
+ String content = matcher.group(2);
+ String value = variables.get(varName);
+ String replacement = (value != null && value.isEmpty() == false) ? content : "";
+ matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+ }
+ matcher.appendTail(sb);
+ result = sb.toString();
+
+ // Replace simple variables: {{var}}
+ for (Map.Entry entry : variables.entrySet()) {
+ String placeholder = "{{" + entry.getKey() + "}}";
+ result = result.replace(placeholder, entry.getValue());
+ }
+
+ return result;
+ }
+
+ /**
+ * Format bytes for display.
+ */
+ public static String formatBytes(long bytes) {
+ if (bytes < 1024) {
+ return bytes + " B";
+ } else if (bytes < 1024 * 1024) {
+ return String.format(Locale.ROOT, "%.1f KB", bytes / 1024.0);
+ } else {
+ return String.format(Locale.ROOT, "%.1f MB", bytes / (1024.0 * 1024.0));
+ }
+ }
+
+ /**
+ * Format time as MM:SS.
+ */
+ public static String formatTime(long minutes, long seconds) {
+ return String.format(Locale.ROOT, "%d:%02d", minutes, seconds);
+ }
+
+ @SuppressForbidden(reason = "System.err is intentional for this interactive manual testing tool")
+ private static PrintStream stderr() {
+ return System.err;
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/README.md b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/README.md
new file mode 100644
index 0000000000000..d957dc87f81d6
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/README.md
@@ -0,0 +1,192 @@
+# Iceberg Test Fixtures
+
+This directory contains pre-built Iceberg metadata and Parquet files used for testing.
+
+## Purpose
+
+These fixtures serve files directly through the S3HttpFixture, eliminating the need for manual test data setup via `addBlobToFixture()` calls. Files placed here are automatically loaded into the fixture's blob storage when tests run.
+
+## Directory Structure
+
+Files in this directory are mapped to S3 paths preserving their structure:
+
+```
+iceberg-fixtures/
+├── README.md # This file
+├── db/ # Database directory
+│ └── table/ # Table directory
+│ ├── metadata/ # Iceberg metadata files
+│ │ ├── v1.metadata.json # Table metadata version 1
+│ │ └── version-hint.text # Current version pointer
+│ └── data/ # Parquet data files
+│ └── part-00000.parquet # Data file
+└── standalone/ # Standalone Parquet files (no Iceberg metadata)
+ └── simple.parquet # Simple Parquet file for direct reading
+```
+
+## S3 Path Mapping
+
+Files are automatically mapped to S3 paths:
+
+- `iceberg-fixtures/db/table/metadata/v1.metadata.json` → `s3://iceberg-test/warehouse/db/table/metadata/v1.metadata.json`
+- `iceberg-fixtures/standalone/simple.parquet` → `s3://iceberg-test/warehouse/standalone/simple.parquet`
+
+## Usage in Tests
+
+### Automatic Loading
+
+All files in this directory are automatically loaded when tests extending `AbstractS3HttpFixtureTest` start:
+
+```java
+public class MyIcebergTest extends AbstractS3HttpFixtureTest {
+
+ public void testReadIcebergTable() throws Exception {
+ // Files from iceberg-fixtures/ are already loaded!
+ Catalog catalog = createCatalog();
+ TableIdentifier tableId = TableIdentifier.of("db", "table");
+ Table table = catalog.loadTable(tableId);
+
+ // Use the table...
+ }
+}
+```
+
+### Manual Addition (Still Supported)
+
+You can still add files programmatically if needed:
+
+```java
+public void testWithDynamicData() {
+ // Add a file at runtime
+ addBlobToFixture("dynamic/test.parquet", parquetBytes);
+
+ // Use it...
+}
+```
+
+## Fixture Categories
+
+### 1. Parquet Format Compatibility
+
+Test different Parquet versions and encodings:
+
+- `parquet-v1/` - Parquet format version 1 files
+- `parquet-v2/` - Parquet format version 2 files
+- `dictionary-encoded/` - Dictionary-encoded columns
+- `plain-encoded/` - Plain-encoded columns
+
+### 2. Edge Cases
+
+Test boundary conditions and special cases:
+
+- `edge-cases/all-nulls.parquet` - File with all null values
+- `edge-cases/empty-columns.parquet` - File with empty columns
+- `edge-cases/large-strings.parquet` - File with large string values
+
+### 3. Iceberg Tables
+
+Complete Iceberg table structures with metadata:
+
+- `db/table/` - Full Iceberg table with metadata and data files
+
+### 4. Regression Tests
+
+Specific files that reproduce known bugs or issues.
+
+## Generating Fixtures
+
+### Using Test Data Generators
+
+The `org.elasticsearch.xpack.esql.iceberg.testdata.generation` package provides utilities for generating test fixtures.
+
+**Note**: These utilities use Parquet's Hadoop-based APIs (`parquet-hadoop`) for writing files. While they import
+Hadoop classes, they use `LocalInputFile`/`LocalOutputFile` which bypass Hadoop's FileSystem and work directly with
+`java.nio.file.Path`. The `Configuration` class is created with `Configuration(false)` to avoid loading Hadoop
+resources and triggering security manager issues.
+
+```java
+// Generate a simple Parquet file
+ParquetWriterUtil.writeParquet(
+ schema,
+ rows,
+ outputFile,
+ ParquetWriterConfig.defaults()
+);
+
+// Generate Iceberg metadata
+IcebergMetadataGenerator.generateMetadata(
+ tableName,
+ parquetFile,
+ outputDir,
+ IcebergMetadataConfig.defaults()
+);
+```
+
+### Using External Tools
+
+You can also generate fixtures using external tools like Apache Spark or Iceberg CLI:
+
+```python
+# Using PySpark
+df = spark.createDataFrame([
+ (1, "Alice", 30),
+ (2, "Bob", 25)
+], ["id", "name", "age"])
+
+df.write.format("parquet").save("simple.parquet")
+```
+
+### Regenerating All Fixtures
+
+To regenerate all fixtures, run the generator tests:
+
+```bash
+./gradlew :x-pack:plugin:esql:test --tests "*IcebergMetadataGeneratorTests"
+```
+
+## Size Guidelines
+
+- Keep individual files under 1MB when possible
+- Total fixture size should stay under 10MB
+- Use compression for text-based metadata files
+- Prefer minimal schemas (3-5 columns) unless testing specific scenarios
+
+## Best Practices
+
+1. **Minimal Data**: Include only the minimum data needed to test the scenario
+2. **Clear Naming**: Use descriptive names that indicate what the fixture tests
+3. **Documentation**: Add comments in test code explaining why each fixture exists
+4. **Regeneration**: Document how to regenerate fixtures if schema changes
+5. **Version Control**: Commit fixtures as binary files (they're small and stable)
+
+## Troubleshooting
+
+### Fixtures Not Loading
+
+If fixtures aren't loading, check:
+
+1. Files are in the correct directory: `src/test/resources/iceberg-fixtures/`
+2. Test class extends `AbstractS3HttpFixtureTest`
+3. Check logs for "Loaded fixtures from iceberg-fixtures directory"
+
+### Path Mapping Issues
+
+If S3 paths don't match expectations:
+
+1. Verify file paths use forward slashes (/)
+2. Check that paths are relative to `iceberg-fixtures/` root
+3. Use `printRequestSummary()` to see actual S3 requests
+
+### File Not Found in Tests
+
+If tests can't find expected files:
+
+1. Verify the S3 path matches the fixture path
+2. Check bucket name is `iceberg-test` and warehouse is `warehouse`
+3. Use `s3Fixture.getHandler().blobs()` to inspect loaded files
+
+## Related Documentation
+
+- [S3 Request Logging](../../../../../../../docs/s3-request-logging.md) - Debugging S3 operations
+- [Iceberg Testing Strategy](../../../../../../../.cursor/plans/iceberg_testing_strategy_decision.md) - Overall testing approach
+- [Test Data Generation](../testdata/generation/) - Programmatic fixture generation
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/data/data.parquet b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/data/data.parquet
new file mode 100644
index 0000000000000..40c723aa7d812
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/data/data.parquet differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro.crc b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro.crc
new file mode 100644
index 0000000000000..2d3a879324bc5
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro.crc differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro.crc b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro.crc
new file mode 100644
index 0000000000000..da1f653c5bee4
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro.crc differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v1.metadata.json.crc b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v1.metadata.json.crc
new file mode 100644
index 0000000000000..85966e2ebd1e5
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v1.metadata.json.crc differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v2.metadata.json.crc b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v2.metadata.json.crc
new file mode 100644
index 0000000000000..a69bcd35d073c
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.v2.metadata.json.crc differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.version-hint.text.crc b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.version-hint.text.crc
new file mode 100644
index 0000000000000..20031206a3b58
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/.version-hint.text.crc differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro
new file mode 100644
index 0000000000000..1d788d9d14f30
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/5947ebd2-0430-4fde-9a42-1b6a58c11c6b-m0.avro
@@ -0,0 +1 @@
+Obj schema {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"birth_date","required":false,"type":"timestamptz"},{"id":2,"name":"emp_no","required":false,"type":"int"},{"id":3,"name":"first_name","required":false,"type":"string"},{"id":4,"name":"gender","required":false,"type":"string"},{"id":5,"name":"hire_date","required":false,"type":"timestamptz"},{"id":6,"name":"languages","required":false,"type":"int"},{"id":7,"name":"languages.long","required":false,"type":"long"},{"id":8,"name":"languages.short","required":false,"type":"int"},{"id":9,"name":"languages.byte","required":false,"type":"int"},{"id":10,"name":"last_name","required":false,"type":"string"},{"id":11,"name":"salary","required":false,"type":"int"},{"id":12,"name":"height","required":false,"type":"double"},{"id":13,"name":"height.float","required":false,"type":"float"},{"id":14,"name":"height.scaled_float","required":false,"type":"double"},{"id":15,"name":"height.half_float","required":false,"type":"float"},{"id":16,"name":"still_hired","required":false,"type":"boolean"},{"id":17,"name":"avg_worked_seconds","required":false,"type":"long"},{"id":18,"name":"job_positions","required":false,"type":{"type":"list","element-id":24,"element":"string","element-required":false}},{"id":19,"name":"is_rehired","required":false,"type":{"type":"list","element-id":25,"element":"boolean","element-required":false}},{"id":20,"name":"salary_change","required":false,"type":{"type":"list","element-id":26,"element":"double","element-required":false}},{"id":21,"name":"salary_change.int","required":false,"type":{"type":"list","element-id":27,"element":"int","element-required":false}},{"id":22,"name":"salary_change.long","required":false,"type":{"type":"list","element-id":28,"element":"long","element-required":false}},{"id":23,"name":"salary_change.keyword","required":false,"type":{"type":"list","element-id":29,"element":"string","element-required":false}}]}avro.schema8{"type":"record","name":"manifest_entry","fields":[{"name":"status","type":"int","field-id":0},{"name":"snapshot_id","type":["null","long"],"default":null,"field-id":1},{"name":"sequence_number","type":["null","long"],"default":null,"field-id":3},{"name":"file_sequence_number","type":["null","long"],"default":null,"field-id":4},{"name":"data_file","type":{"type":"record","name":"r2","fields":[{"name":"content","type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes","field-id":134},{"name":"file_path","type":"string","doc":"Location URI with FS scheme","field-id":100},{"name":"file_format","type":"string","doc":"File format name: avro, orc, or parquet","field-id":101},{"name":"partition","type":{"type":"record","name":"r102","fields":[]},"doc":"Partition data tuple, schema based on the partition spec","field-id":102},{"name":"record_count","type":"long","doc":"Number of records in the file","field-id":103},{"name":"file_size_in_bytes","type":"long","doc":"Total file size in bytes","field-id":104},{"name":"column_sizes","type":["null",{"type":"array","items":{"type":"record","name":"k117_v118","fields":[{"name":"key","type":"int","field-id":117},{"name":"value","type":"long","field-id":118}]},"logicalType":"map"}],"doc":"Map of column id to total size on disk","default":null,"field-id":108},{"name":"value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k119_v120","fields":[{"name":"key","type":"int","field-id":119},{"name":"value","type":"long","field-id":120}]},"logicalType":"map"}],"doc":"Map of column id to total count, including null and NaN","default":null,"field-id":109},{"name":"null_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k121_v122","fields":[{"name":"key","type":"int","field-id":121},{"name":"value","type":"long","field-id":122}]},"logicalType":"map"}],"doc":"Map of column id to null value count","default":null,"field-id":110},{"name":"nan_value_counts","type":["null",{"type":"array","items":{"type":"record","name":"k138_v139","fields":[{"name":"key","type":"int","field-id":138},{"name":"value","type":"long","field-id":139}]},"logicalType":"map"}],"doc":"Map of column id to number of NaN values in the column","default":null,"field-id":137},{"name":"lower_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k126_v127","fields":[{"name":"key","type":"int","field-id":126},{"name":"value","type":"bytes","field-id":127}]},"logicalType":"map"}],"doc":"Map of column id to lower bound","default":null,"field-id":125},{"name":"upper_bounds","type":["null",{"type":"array","items":{"type":"record","name":"k129_v130","fields":[{"name":"key","type":"int","field-id":129},{"name":"value","type":"bytes","field-id":130}]},"logicalType":"map"}],"doc":"Map of column id to upper bound","default":null,"field-id":128},{"name":"key_metadata","type":["null","bytes"],"doc":"Encryption key metadata blob","default":null,"field-id":131},{"name":"split_offsets","type":["null",{"type":"array","items":"long","element-id":133}],"doc":"Splittable offsets","default":null,"field-id":132},{"name":"equality_ids","type":["null",{"type":"array","items":"int","element-id":136}],"doc":"Equality comparison field IDs","default":null,"field-id":135},{"name":"sort_order_id","type":["null","int"],"doc":"Sort order ID","default":null,"field-id":140},{"name":"referenced_data_file","type":["null","string"],"doc":"Fully qualified location (URI with FS scheme) of a data file that all deletes reference","default":null,"field-id":143}]},"field-id":2}]}avro.codecdeflate format-version2"partition-spec-id0 iceberg.schema.{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position deletes, 2=equality deletes"},{"id":100,"name":"file_path","required":true,"type":"string","doc":"Location URI with FS scheme"},{"id":101,"name":"file_format","required":true,"type":"string","doc":"File format name: avro, orc, or parquet"},{"id":102,"name":"partition","required":true,"type":{"type":"struct","fields":[]},"doc":"Partition data tuple, schema based on the partition spec"},{"id":103,"name":"record_count","required":true,"type":"long","doc":"Number of records in the file"},{"id":104,"name":"file_size_in_bytes","required":true,"type":"long","doc":"Total file size in bytes"},{"id":108,"name":"column_sizes","required":false,"type":{"type":"map","key-id":117,"key":"int","value-id":118,"value":"long","value-required":true},"doc":"Map of column id to total size on disk"},{"id":109,"name":"value_counts","required":false,"type":{"type":"map","key-id":119,"key":"int","value-id":120,"value":"long","value-required":true},"doc":"Map of column id to total count, including null and NaN"},{"id":110,"name":"null_value_counts","required":false,"type":{"type":"map","key-id":121,"key":"int","value-id":122,"value":"long","value-required":true},"doc":"Map of column id to null value count"},{"id":137,"name":"nan_value_counts","required":false,"type":{"type":"map","key-id":138,"key":"int","value-id":139,"value":"long","value-required":true},"doc":"Map of column id to number of NaN values in the column"},{"id":125,"name":"lower_bounds","required":false,"type":{"type":"map","key-id":126,"key":"int","value-id":127,"value":"binary","value-required":true},"doc":"Map of column id to lower bound"},{"id":128,"name":"upper_bounds","required":false,"type":{"type":"map","key-id":129,"key":"int","value-id":130,"value":"binary","value-required":true},"doc":"Map of column id to upper bound"},{"id":131,"name":"key_metadata","required":false,"type":"binary","doc":"Encryption key metadata blob"},{"id":132,"name":"split_offsets","required":false,"type":{"type":"list","element-id":133,"element":"long","element-required":true},"doc":"Splittable offsets"},{"id":135,"name":"equality_ids","required":false,"type":{"type":"list","element-id":136,"element":"int","element-required":true},"doc":"Equality comparison field IDs"},{"id":140,"name":"sort_order_id","required":false,"type":"int","doc":"Sort order ID"},{"id":143,"name":"referenced_data_file","required":false,"type":"string","doc":"Fully qualified location (URI with FS scheme) of a data file that all deletes reference"}]}}]} partition-spec[]contentdata bD'D cbZ2ՃVgd``+6LNMJ-J-I-./O,J/-NO-ɯLM-OI,IzE%|A!'=L bD'D
\ No newline at end of file
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro
new file mode 100644
index 0000000000000..d27b98a56726d
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v1.metadata.json b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v1.metadata.json
new file mode 100644
index 0000000000000..0af7d857a8ce6
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"3ca7afdd-bd7e-4706-b0aa-2f2d50561ca2","location":"s3://iceberg-test/warehouse/employees","last-sequence-number":0,"last-updated-ms":1769593830928,"last-column-id":29,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"birth_date","required":false,"type":"timestamptz"},{"id":2,"name":"emp_no","required":false,"type":"int"},{"id":3,"name":"first_name","required":false,"type":"string"},{"id":4,"name":"gender","required":false,"type":"string"},{"id":5,"name":"hire_date","required":false,"type":"timestamptz"},{"id":6,"name":"languages","required":false,"type":"int"},{"id":7,"name":"languages.long","required":false,"type":"long"},{"id":8,"name":"languages.short","required":false,"type":"int"},{"id":9,"name":"languages.byte","required":false,"type":"int"},{"id":10,"name":"last_name","required":false,"type":"string"},{"id":11,"name":"salary","required":false,"type":"int"},{"id":12,"name":"height","required":false,"type":"double"},{"id":13,"name":"height.float","required":false,"type":"float"},{"id":14,"name":"height.scaled_float","required":false,"type":"double"},{"id":15,"name":"height.half_float","required":false,"type":"float"},{"id":16,"name":"still_hired","required":false,"type":"boolean"},{"id":17,"name":"avg_worked_seconds","required":false,"type":"long"},{"id":18,"name":"job_positions","required":false,"type":{"type":"list","element-id":24,"element":"string","element-required":false}},{"id":19,"name":"is_rehired","required":false,"type":{"type":"list","element-id":25,"element":"boolean","element-required":false}},{"id":20,"name":"salary_change","required":false,"type":{"type":"list","element-id":26,"element":"double","element-required":false}},{"id":21,"name":"salary_change.int","required":false,"type":{"type":"list","element-id":27,"element":"int","element-required":false}},{"id":22,"name":"salary_change.long","required":false,"type":{"type":"list","element-id":28,"element":"long","element-required":false}},{"id":23,"name":"salary_change.keyword","required":false,"type":{"type":"list","element-id":29,"element":"string","element-required":false}}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":-1,"refs":{},"snapshots":[],"statistics":[],"partition-statistics":[],"snapshot-log":[],"metadata-log":[]}
\ No newline at end of file
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v2.metadata.json b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v2.metadata.json
new file mode 100644
index 0000000000000..29564c09b594a
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"3ca7afdd-bd7e-4706-b0aa-2f2d50561ca2","location":"s3://iceberg-test/warehouse/employees","last-sequence-number":1,"last-updated-ms":1769593831391,"last-column-id":29,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"birth_date","required":false,"type":"timestamptz"},{"id":2,"name":"emp_no","required":false,"type":"int"},{"id":3,"name":"first_name","required":false,"type":"string"},{"id":4,"name":"gender","required":false,"type":"string"},{"id":5,"name":"hire_date","required":false,"type":"timestamptz"},{"id":6,"name":"languages","required":false,"type":"int"},{"id":7,"name":"languages.long","required":false,"type":"long"},{"id":8,"name":"languages.short","required":false,"type":"int"},{"id":9,"name":"languages.byte","required":false,"type":"int"},{"id":10,"name":"last_name","required":false,"type":"string"},{"id":11,"name":"salary","required":false,"type":"int"},{"id":12,"name":"height","required":false,"type":"double"},{"id":13,"name":"height.float","required":false,"type":"float"},{"id":14,"name":"height.scaled_float","required":false,"type":"double"},{"id":15,"name":"height.half_float","required":false,"type":"float"},{"id":16,"name":"still_hired","required":false,"type":"boolean"},{"id":17,"name":"avg_worked_seconds","required":false,"type":"long"},{"id":18,"name":"job_positions","required":false,"type":{"type":"list","element-id":24,"element":"string","element-required":false}},{"id":19,"name":"is_rehired","required":false,"type":{"type":"list","element-id":25,"element":"boolean","element-required":false}},{"id":20,"name":"salary_change","required":false,"type":{"type":"list","element-id":26,"element":"double","element-required":false}},{"id":21,"name":"salary_change.int","required":false,"type":{"type":"list","element-id":27,"element":"int","element-required":false}},{"id":22,"name":"salary_change.long","required":false,"type":{"type":"list","element-id":28,"element":"long","element-required":false}},{"id":23,"name":"salary_change.keyword","required":false,"type":{"type":"list","element-id":29,"element":"string","element-required":false}}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"write.parquet.compression-codec":"zstd"},"current-snapshot-id":5740414668264810322,"refs":{"main":{"snapshot-id":5740414668264810322,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":5740414668264810322,"timestamp-ms":1769593831391,"summary":{"operation":"append","added-data-files":"1","added-records":"100","added-files-size":"14483","changed-partition-count":"1","total-records":"100","total-files-size":"14483","total-data-files":"1","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","iceberg-version":"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"},"manifest-list":"s3://iceberg-test/warehouse/employees/metadata/snap-5740414668264810322-1-5947ebd2-0430-4fde-9a42-1b6a58c11c6b.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1769593831391,"snapshot-id":5740414668264810322}],"metadata-log":[{"timestamp-ms":1769593830928,"metadata-file":"s3://iceberg-test/warehouse/employees/metadata/v1.metadata.json"}]}
\ No newline at end of file
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/version-hint.text b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/version-hint.text
new file mode 100644
index 0000000000000..d8263ee986059
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/employees/metadata/version-hint.text
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet
new file mode 100644
index 0000000000000..40c723aa7d812
Binary files /dev/null and b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet differ
diff --git a/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/interactive-fixture-messages.txt b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/interactive-fixture-messages.txt
new file mode 100644
index 0000000000000..d2f0f5ccbca32
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/qa/src/javaRestTest/resources/interactive-fixture-messages.txt
@@ -0,0 +1,163 @@
+# Interactive Fixture Messages
+# Template file for InteractiveFixtureIT output
+# Variables are replaced using {{variable_name}} syntax
+
+[banner]
+================================================================================
+ ESQL EXTERNAL COMMAND - INTERACTIVE FIXTURE MODE
+================================================================================
+
+[cluster_info]
+
+📊 ELASTICSEARCH CLUSTER
+ URL: {{es_url}}
+ Security: Disabled (no authentication required)
+ License: Trial
+ S3 Endpoint: {{s3_endpoint}}
+
+[fixture_info]
+
+🗄️ S3 HTTP FIXTURE
+ URL: {{fixture_url}}
+ Bucket: {{bucket}}
+ Warehouse: {{warehouse}}
+ Access Key: {{access_key}}
+ Secret Key: {{secret_key}}
+ Protocol: HTTP (no TLS)
+ Port: {{port}} (randomly assigned)
+
+ ℹ️ IMPORTANT: Both protocols use the SAME port!
+ • S3 API: s3://{{bucket}}/{{warehouse}}/... → {{fixture_url}} (via S3 SDK)
+ • HTTP API: {{fixture_url}}/{{bucket}}/{{warehouse}}/... (direct)
+
+ The fixture is an HTTP server that implements the S3 API.
+ S3 URLs are translated by ES's S3 client into HTTP requests to this port.
+
+[fixtures_header]
+
+📁 AVAILABLE FIXTURES
+ Total files: {{total_files}}
+ Parquet files: {{parquet_count}}
+ Metadata files: {{metadata_count}}
+{{#other_count}} Other files: {{other_count}}{{/other_count}}
+
+[fixtures_show_all]
+
+ All loaded fixtures:
+
+[fixtures_show_key]
+
+ Key fixtures:
+
+[fixtures_footer]
+
+ (Use -Dtests.fixture.show_blobs=true to see all fixtures)
+
+[example_queries]
+
+🔍 EXAMPLE QUERIES (New WITH Syntax)
+
+ Method 1: S3 Protocol with WITH clause (recommended)
+ ────────────────────────────────────────────────────
+ curl -X POST "{{es_url}}/_query?format=txt" \
+ -H 'Content-Type: application/json' -d'{
+ "query": "EXTERNAL \"s3://{{bucket}}/{{warehouse}}/standalone/employees.parquet\" WITH { \"endpoint\": \"{{s3_endpoint}}\", \"access_key\": \"{{access_key}}\", \"secret_key\": \"{{secret_key}}\" } | LIMIT 5"
+ }'
+
+ Method 2: HTTP Protocol with WITH clause (direct URL)
+ ──────────────────────────────────────────────────────
+ curl -X POST "{{es_url}}/_query?format=txt" \
+ -H 'Content-Type: application/json' -d'{
+ "query": "EXTERNAL \"{{fixture_url}}/{{bucket}}/{{warehouse}}/standalone/employees.parquet\" WITH { \"endpoint\": \"{{s3_endpoint}}\", \"access_key\": \"{{access_key}}\", \"secret_key\": \"{{secret_key}}\" } | LIMIT 5"
+ }'
+
+ Kibana Dev Console (S3 Protocol)
+ ─────────────────────────────────
+ POST /_query?format=txt
+ {
+ "query": "EXTERNAL \"s3://{{bucket}}/{{warehouse}}/standalone/employees.parquet\" WITH { \"endpoint\": \"{{s3_endpoint}}\", \"access_key\": \"{{access_key}}\", \"secret_key\": \"{{secret_key}}\" } | LIMIT 5"
+ }
+
+ More Examples
+ ─────────────
+ # Filter employees (multiline for readability)
+ EXTERNAL "s3://{{bucket}}/{{warehouse}}/standalone/employees.parquet"
+ WITH {
+ "endpoint": "{{s3_endpoint}}",
+ "access_key": "{{access_key}}",
+ "secret_key": "{{secret_key}}"
+ }
+ | WHERE gender == "F" AND salary > 60000
+ | KEEP first_name, last_name, salary
+ | SORT salary DESC
+ | LIMIT 10
+
+ # Aggregate by gender
+ EXTERNAL "s3://{{bucket}}/{{warehouse}}/standalone/employees.parquet"
+ WITH {
+ "endpoint": "{{s3_endpoint}}",
+ "access_key": "{{access_key}}",
+ "secret_key": "{{secret_key}}"
+ }
+ | STATS avg_salary = AVG(salary), count = COUNT(*) BY gender
+
+ # Using HTTP protocol (no S3 credentials needed for HTTP direct access)
+ EXTERNAL "{{fixture_url}}/{{bucket}}/{{warehouse}}/standalone/employees.parquet"
+ | LIMIT 5
+
+[wait_indefinite]
+
+⏳ INTERACTIVE SESSION
+ Fixture and cluster are now running
+ Running indefinitely - Press Ctrl+C to stop
+ (Set time limit with: -Dtests.fixture.wait_minutes=N)
+
+────────────────────────────────────────────────────────────────────────────────
+
+[wait_timed]
+
+⏳ INTERACTIVE SESSION
+ Fixture and cluster are now running
+ Waiting {{wait_minutes}} minute(s) for manual testing...
+ (Run indefinitely with: -Dtests.fixture.wait_minutes=0)
+
+────────────────────────────────────────────────────────────────────────────────
+
+[progress_indefinite]
+ ⏱️ Running for: {{elapsed_time}} (Press Ctrl+C to stop)
+
+[progress_timed]
+ ⏱️ Time remaining: {{remaining_time}}
+
+[request_log_header]
+
+────────────────────────────────────────────────────────────────────────────────
+📝 S3 REQUEST LOG SUMMARY
+────────────────────────────────────────────────────────────────────────────────
+
+[request_log_empty]
+
+ No S3 requests were made during this session.
+ (This is expected if you didn't run any queries)
+
+[request_log_summary]
+
+ Total requests: {{total_requests}}
+
+ Requests by type:
+
+[request_log_paths]
+
+ Unique paths accessed:
+
+[request_log_paths_truncated]
+ ... (showing first 20 paths)
+
+[shutdown]
+
+================================================================================
+ SHUTTING DOWN
+================================================================================
+
+ Fixture and cluster will now stop.
+ Test completed successfully.
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapter.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapter.java
new file mode 100644
index 0000000000000..7d90ce3fbfa22
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapter.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StaticTableOperations;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.aws.s3.S3FileIO;
+import org.apache.iceberg.io.FileIO;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.IOException;
+
+/**
+ * Adapter for accessing Iceberg catalog and table metadata.
+ * Provides a simplified interface for resolving Iceberg tables.
+ *
+ * This implementation uses Iceberg's StaticTableOperations with S3FileIO,
+ * avoiding Hadoop dependencies and security manager issues.
+ */
+public class IcebergCatalogAdapter {
+
+ private static final String SOURCE_TYPE_ICEBERG = "iceberg";
+ private static final String METADATA_DIR = "metadata";
+ private static final String METADATA_FILE_EXTENSION = ".metadata.json";
+
+ /**
+ * Resolve Iceberg table metadata from a table path.
+ * Uses StaticTableOperations with S3FileIO instead of HadoopCatalog.
+ *
+ * @param tablePath the S3 path to the Iceberg table
+ * @param s3Config S3 configuration (credentials, endpoint, etc.)
+ * @return IcebergTableMetadata with resolved schema
+ * @throws Exception if table cannot be resolved
+ */
+ public static IcebergTableMetadata resolveTable(String tablePath, S3Configuration s3Config) throws Exception {
+ // Create S3FileIO for accessing table metadata
+ S3FileIO fileIO = S3FileIOFactory.create(s3Config);
+
+ try {
+ // Find the latest metadata file
+ String metadataLocation = findLatestMetadataFile(tablePath, fileIO);
+
+ // Load table using StaticTableOperations
+ StaticTableOperations ops = new StaticTableOperations(metadataLocation, fileIO);
+ Table table = new BaseTable(ops, tablePath);
+ Schema schema = table.schema();
+
+ // Pass the metadata location so we can recreate the table later if needed
+ return new IcebergTableMetadata(tablePath, schema, s3Config, SOURCE_TYPE_ICEBERG, metadataLocation);
+ } finally {
+ // Close FileIO to release resources - use IOUtils which logs suppressed exceptions
+ IOUtils.closeWhileHandlingException(fileIO);
+ }
+ }
+
+ /**
+ * Find the latest metadata file in the table's metadata directory.
+ * Iceberg tables store metadata in versioned JSON files like v1.metadata.json, v2.metadata.json, etc.
+ *
+ * Since FileIO doesn't have a listPrefix method, we try common version numbers.
+ * This is a simplified approach that works for test fixtures and small tables.
+ * For production, consider using a catalog that tracks the current metadata location.
+ *
+ * @param tablePath the base path to the Iceberg table
+ * @param fileIO the FileIO to use for checking file existence
+ * @return the full path to the latest metadata file
+ * @throws IOException if no metadata files found
+ */
+ private static String findLatestMetadataFile(String tablePath, FileIO fileIO) throws IOException {
+ // Ensure tablePath ends with /
+ String normalizedPath = tablePath.endsWith("/") ? tablePath : tablePath + "/";
+ String metadataDir = normalizedPath + METADATA_DIR + "/";
+
+ // First, try to read version-hint.text which points to the current metadata version
+ // This is the most reliable approach as it's maintained by Iceberg
+ String versionHintPath = metadataDir + "version-hint.text";
+ try {
+ org.apache.iceberg.io.InputFile versionHintFile = fileIO.newInputFile(versionHintPath);
+ if (versionHintFile.exists()) {
+ // Read the version number from the hint file
+ try (java.io.InputStream is = versionHintFile.newStream()) {
+ String versionStr = new String(is.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).trim();
+ int version = Integer.parseInt(versionStr);
+ String metadataPath = metadataDir + "v" + version + METADATA_FILE_EXTENSION;
+ // Verify the metadata file exists
+ org.apache.iceberg.io.InputFile metadataFile = fileIO.newInputFile(metadataPath);
+ if (metadataFile.exists()) {
+ return metadataPath;
+ }
+ }
+ }
+ } catch (Exception e) {
+ // Version hint doesn't exist or couldn't be read, fall through to scan
+ }
+
+ // Fallback: Try to find metadata files by checking common version numbers
+ // Start from a reasonable max version and work backwards
+ for (int version = 100; version >= 1; version--) {
+ String metadataPath = metadataDir + "v" + version + METADATA_FILE_EXTENSION;
+ try {
+ org.apache.iceberg.io.InputFile inputFile = fileIO.newInputFile(metadataPath);
+ // Actually check if the file exists - newInputFile() alone doesn't verify existence
+ if (inputFile.exists()) {
+ return metadataPath;
+ }
+ } catch (Exception e) {
+ // Error checking this version, try next
+ }
+ }
+
+ throw new IOException("No metadata files found in " + metadataDir + ". Tried version-hint.text and versions 1-100");
+ }
+
+ /**
+ * Extract version number from a metadata filename.
+ * For example: "s3://bucket/table/metadata/v123.metadata.json" -> 123
+ *
+ * @param path the full path to the metadata file
+ * @return the version number, or 0 if it cannot be parsed
+ */
+ static int extractVersionNumber(String path) {
+ try {
+ // Get filename from path
+ int lastSlash = path.lastIndexOf('/');
+ String filename = lastSlash >= 0 ? path.substring(lastSlash + 1) : path;
+
+ // Remove "v" prefix and ".metadata.json" suffix
+ if (filename.startsWith("v") && filename.endsWith(METADATA_FILE_EXTENSION)) {
+ String versionStr = filename.substring(1, filename.length() - METADATA_FILE_EXTENSION.length());
+ return Integer.parseInt(versionStr);
+ }
+ } catch (NumberFormatException e) {
+ // If parsing fails, return 0
+ }
+ return 0;
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergDataSourcePlugin.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergDataSourcePlugin.java
new file mode 100644
index 0000000000000..a71f452c6e823
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergDataSourcePlugin.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin;
+import org.elasticsearch.xpack.esql.datasources.spi.TableCatalogFactory;
+
+import java.util.Map;
+
+/**
+ * Data source plugin that provides Iceberg table catalog support for ESQL external data sources.
+ *
+ *
This plugin provides:
+ *
+ * Iceberg table catalog for reading Iceberg tables from S3
+ * Schema discovery from Iceberg metadata
+ * Predicate pushdown for efficient filtering
+ * Vectorized reading using Arrow format
+ *
+ *
+ * The Iceberg implementation uses:
+ *
+ * Iceberg's StaticTableOperations for metadata access
+ * S3FileIO for S3 storage access
+ * ArrowReader for efficient vectorized columnar data reading
+ *
+ *
+ * Heavy dependencies (Iceberg, Arrow, Parquet, AWS SDK) are isolated in this module
+ * to avoid jar hell issues in the core ESQL plugin.
+ */
+public class IcebergDataSourcePlugin extends Plugin implements DataSourcePlugin {
+
+ @Override
+ public Map tableCatalogs(Settings settings) {
+ return Map.of("iceberg", s -> new IcebergTableCatalog());
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFilters.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFilters.java
new file mode 100644
index 0000000000000..2ac4d2ce4611f
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFilters.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.expression.predicate.Range;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNull;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iceberg.expressions.Expressions.and;
+import static org.apache.iceberg.expressions.Expressions.equal;
+import static org.apache.iceberg.expressions.Expressions.greaterThan;
+import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.in;
+import static org.apache.iceberg.expressions.Expressions.isNull;
+import static org.apache.iceberg.expressions.Expressions.lessThan;
+import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.not;
+import static org.apache.iceberg.expressions.Expressions.notEqual;
+import static org.apache.iceberg.expressions.Expressions.notNull;
+import static org.apache.iceberg.expressions.Expressions.or;
+import static org.elasticsearch.xpack.esql.expression.Foldables.literalValueOf;
+
+/**
+ * Converts ESQL expressions to Iceberg filter expressions for predicate pushdown.
+ * Supports comparison operators, logical operators, and null checks.
+ */
+public class IcebergPushdownFilters {
+
+ /**
+ * Convert an ESQL expression to an Iceberg filter expression.
+ * Returns null if the expression cannot be converted (unsupported predicate).
+ */
+ public static org.apache.iceberg.expressions.Expression convert(Expression esqlExpr) {
+ // Binary comparisons: field op value
+ if (esqlExpr instanceof EsqlBinaryComparison bc && bc.left() instanceof NamedExpression ne && bc.right().foldable()) {
+ String fieldName = ne.name();
+ Object value = convertValue(literalValueOf(bc.right()));
+
+ return switch (bc) {
+ case Equals ignored -> equal(fieldName, value);
+ case NotEquals ignored -> notEqual(fieldName, value);
+ case LessThan ignored -> lessThan(fieldName, value);
+ case LessThanOrEqual ignored -> lessThanOrEqual(fieldName, value);
+ case GreaterThan ignored -> greaterThan(fieldName, value);
+ case GreaterThanOrEqual ignored -> greaterThanOrEqual(fieldName, value);
+ default -> null;
+ };
+ }
+
+ // In: field IN (value1, value2, ...)
+ if (esqlExpr instanceof In inExpr && inExpr.value() instanceof NamedExpression ne) {
+ List list = inExpr.list();
+ List values = new ArrayList<>(list.size());
+ for (Expression expr : list) {
+ if (expr.foldable() == false) {
+ return null;
+ }
+ values.add(convertValue(literalValueOf(expr)));
+ }
+ return in(ne.name(), values);
+ }
+
+ // IsNull: field IS NULL
+ if (esqlExpr instanceof IsNull isNullExpr && isNullExpr.field() instanceof NamedExpression ne) {
+ return isNull(ne.name());
+ }
+
+ // IsNotNull: field IS NOT NULL
+ if (esqlExpr instanceof IsNotNull isNotNullExpr && isNotNullExpr.field() instanceof NamedExpression ne) {
+ return notNull(ne.name());
+ }
+
+ // Range: lower <= field <= upper (or variations with < and >)
+ if (esqlExpr instanceof Range range
+ && range.value() instanceof NamedExpression ne
+ && range.lower().foldable()
+ && range.upper().foldable()) {
+ String fieldName = ne.name();
+ Object lowerValue = convertValue(literalValueOf(range.lower()));
+ Object upperValue = convertValue(literalValueOf(range.upper()));
+
+ org.apache.iceberg.expressions.Expression lowerBound = range.includeLower()
+ ? greaterThanOrEqual(fieldName, lowerValue)
+ : greaterThan(fieldName, lowerValue);
+ org.apache.iceberg.expressions.Expression upperBound = range.includeUpper()
+ ? lessThanOrEqual(fieldName, upperValue)
+ : lessThan(fieldName, upperValue);
+
+ return and(lowerBound, upperBound);
+ }
+
+ // Binary logical operators: AND, OR
+ if (esqlExpr instanceof BinaryLogic bl) {
+ org.apache.iceberg.expressions.Expression left = convert(bl.left());
+ org.apache.iceberg.expressions.Expression right = convert(bl.right());
+ if (left != null && right != null) {
+ return switch (bl) {
+ case And ignored -> and(left, right);
+ case Or ignored -> or(left, right);
+ default -> null;
+ };
+ }
+ return null;
+ }
+
+ // Not: NOT expr
+ if (esqlExpr instanceof Not notExpr) {
+ org.apache.iceberg.expressions.Expression inner = convert(notExpr.field());
+ if (inner != null) {
+ return not(inner);
+ }
+ return null;
+ }
+
+ return null;
+ }
+
+ private static Object convertValue(Object value) {
+ return BytesRefs.toString(value);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergSourceOperatorFactory.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergSourceOperatorFactory.java
new file mode 100644
index 0000000000000..42ec8cc55433b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergSourceOperatorFactory.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.iceberg.CombinedScanTask;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableScan;
+import org.apache.iceberg.arrow.vectorized.ArrowReader;
+import org.apache.iceberg.arrow.vectorized.ColumnVector;
+import org.apache.iceberg.arrow.vectorized.ColumnarBatch;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.CloseableIterator;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.Executor;
+import java.util.function.Supplier;
+
+/**
+ * Factory for creating async source operators for Iceberg tables.
+ *
+ * This factory creates operators that read data from Iceberg tables or Parquet files using:
+ *
+ * Iceberg's {@link ArrowReader} for efficient vectorized columnar data reading
+ * Arrow format ({@link VectorSchemaRoot}) for in-memory representation
+ * Background executor thread to avoid blocking the Driver during S3 I/O
+ *
+ *
+ * Each operator gets:
+ *
+ * A shared buffer for pages
+ * A background reader task that fills the buffer
+ * An executor to run the background task
+ *
+ */
+public class IcebergSourceOperatorFactory implements SourceOperator.SourceOperatorFactory {
+
+ private final Executor executor;
+ private final String tablePath;
+ private final S3Configuration s3Config;
+ private final String sourceType;
+ private final Expression filter;
+ private final Schema schema;
+ private final List attributes;
+ private final int pageSize;
+ private final int maxBufferSize;
+
+ /**
+ * @param executor Executor for running background S3/Iceberg reads
+ * @param tablePath Path to Iceberg table or Parquet file
+ * @param s3Config S3 configuration (credentials, endpoint, region)
+ * @param sourceType Type of source ("iceberg" or "parquet")
+ * @param filter Iceberg filter expression (nullable)
+ * @param schema Iceberg schema
+ * @param attributes ESQL attributes (schema)
+ * @param pageSize Number of rows per page (batch size for Vectorized Reader)
+ * @param maxBufferSize Maximum number of pages to buffer
+ */
+ public IcebergSourceOperatorFactory(
+ Executor executor,
+ String tablePath,
+ S3Configuration s3Config,
+ String sourceType,
+ Expression filter,
+ Schema schema,
+ List attributes,
+ int pageSize,
+ int maxBufferSize
+ ) {
+ this.executor = executor;
+ this.tablePath = tablePath;
+ this.s3Config = s3Config;
+ this.sourceType = sourceType;
+ this.filter = filter;
+ this.schema = schema;
+ this.attributes = attributes;
+ this.pageSize = pageSize;
+ this.maxBufferSize = maxBufferSize;
+ }
+
+ @Override
+ public SourceOperator get(DriverContext driverContext) {
+ // TODO: Implement async source operator creation
+ // This requires integration with the ESQL async operator infrastructure.
+ // For now, the Iceberg plugin provides TableCatalog functionality for schema discovery.
+ // Full data reading support will be added in a future iteration.
+ throw new UnsupportedOperationException(
+ "Direct Iceberg source operator creation is not yet supported. "
+ + "Use the generic async operator factory via OperatorFactoryRegistry."
+ );
+ }
+
+ /**
+ * Create a data supplier that provides Iceberg data using Vectorized Reader with Arrow format.
+ * This supplier lazily initializes the Iceberg table scan and reader.
+ */
+ private Supplier> createDataSupplier() {
+ return () -> {
+ try {
+ return createIcebergTableReader();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create Iceberg data reader for: " + tablePath, e);
+ }
+ };
+ }
+
+ /**
+ * Create a reader for an Iceberg table using Iceberg's ArrowReader.
+ * Returns VectorSchemaRoot batches by converting ColumnarBatch from ArrowReader.
+ */
+ private CloseableIterable createIcebergTableReader() throws Exception {
+ // Recreate the table from metadata location
+ // Note: We need to recreate it here because we can't keep FileIO open across the entire query
+ IcebergTableMetadata metadata = IcebergCatalogAdapter.resolveTable(tablePath, s3Config);
+
+ // Recreate the Table object for scanning
+ org.apache.iceberg.aws.s3.S3FileIO fileIO = S3FileIOFactory.create(s3Config);
+ org.apache.iceberg.StaticTableOperations ops = new org.apache.iceberg.StaticTableOperations(metadata.metadataLocation(), fileIO);
+ Table table = new org.apache.iceberg.BaseTable(ops, tablePath);
+
+ // Use planWith() to set a direct (current-thread) executor, avoiding the default ThreadPool/shutdown hooks
+ TableScan scan = table.newScan().planWith(org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE);
+
+ if (filter != null) {
+ scan = scan.filter(filter);
+ }
+
+ // Project only the columns we need based on attributes
+ if (attributes != null && attributes.isEmpty() == false) {
+ List columnNames = new ArrayList<>();
+ for (Attribute attr : attributes) {
+ columnNames.add(attr.name());
+ }
+ scan = scan.select(columnNames);
+ }
+
+ // Get the scan tasks - use planFiles() to get individual file tasks
+ CloseableIterable fileTasks = scan.planFiles();
+
+ // Convert FileScanTasks to CombinedScanTasks (each file as its own combined task)
+ CloseableIterable tasks = org.apache.iceberg.io.CloseableIterable.transform(
+ fileTasks,
+ fileTask -> new org.apache.iceberg.BaseCombinedScanTask(java.util.Collections.singletonList(fileTask))
+ );
+
+ // Create ArrowReader with the specified page size (batch size)
+ // reuseContainers=false for safety (true could reuse buffers across batches)
+ ArrowReader arrowReader = new ArrowReader(scan, pageSize, /* reuseContainers */ false);
+
+ // Create a buffer allocator for Arrow memory management
+ BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+
+ // Open the reader to get an iterator of ColumnarBatch
+ CloseableIterator batchIterator = arrowReader.open(tasks);
+
+ // Wrap the ColumnarBatch iterator to return VectorSchemaRoot
+ return new ColumnarBatchToVectorSchemaRootIterable(batchIterator, allocator, arrowReader);
+ }
+
+ @Override
+ public String describe() {
+ return "IcebergSourceOperator[path=" + tablePath + ", pageSize=" + pageSize + ", bufferSize=" + maxBufferSize + "]";
+ }
+
+ /**
+ * Adapter that converts Iceberg's ColumnarBatch iterator to VectorSchemaRoot iterator.
+ * This bridges between Iceberg's vectorized reader format and the Arrow format expected by ESQL.
+ */
+ private static class ColumnarBatchToVectorSchemaRootIterable implements CloseableIterable {
+ private final CloseableIterator batchIterator;
+ private final BufferAllocator allocator;
+ private final ArrowReader arrowReader;
+
+ ColumnarBatchToVectorSchemaRootIterable(
+ CloseableIterator batchIterator,
+ BufferAllocator allocator,
+ ArrowReader arrowReader
+ ) {
+ this.batchIterator = batchIterator;
+ this.allocator = allocator;
+ this.arrowReader = arrowReader;
+ }
+
+ @Override
+ public CloseableIterator iterator() {
+ return new CloseableIterator() {
+ @Override
+ public boolean hasNext() {
+ return batchIterator.hasNext();
+ }
+
+ @Override
+ public VectorSchemaRoot next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+
+ ColumnarBatch batch = batchIterator.next();
+ return convertColumnarBatchToVectorSchemaRoot(batch);
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ batchIterator.close();
+ } finally {
+ try {
+ arrowReader.close();
+ } finally {
+ allocator.close();
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public void close() throws IOException {
+ iterator().close();
+ }
+
+ /**
+ * Convert a ColumnarBatch (Iceberg's format) to VectorSchemaRoot (Arrow's format).
+ * The ColumnarBatch wraps Arrow FieldVectors via ColumnVector wrappers.
+ */
+ private VectorSchemaRoot convertColumnarBatchToVectorSchemaRoot(ColumnarBatch batch) {
+ int numRows = batch.numRows();
+ int numColumns = batch.numCols();
+
+ // Extract the underlying Arrow FieldVectors from the ColumnVector wrappers
+ List fieldVectors = new ArrayList<>(numColumns);
+ for (int col = 0; col < numColumns; col++) {
+ ColumnVector columnVector = batch.column(col);
+ // Get the underlying Arrow FieldVector from the ColumnVector wrapper
+ FieldVector fieldVector = columnVector.getFieldVector();
+ fieldVectors.add(fieldVector);
+ }
+
+ // Create VectorSchemaRoot from the field vectors
+ // Note: We pass the vectors directly; they are already allocated and populated
+ return new VectorSchemaRoot(fieldVectors);
+ }
+ }
+
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableCatalog.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableCatalog.java
new file mode 100644
index 0000000000000..798f3de6dc194
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableCatalog.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StaticTableOperations;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableScan;
+import org.apache.iceberg.aws.s3.S3FileIO;
+import org.apache.iceberg.io.CloseableIterable;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.xpack.esql.datasources.spi.SourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.TableCatalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Iceberg table catalog implementation.
+ * Provides metadata resolution and scan planning for Iceberg tables stored in S3.
+ */
+public class IcebergTableCatalog implements TableCatalog {
+
+ private static final String CATALOG_TYPE = "iceberg";
+
+ @Override
+ public String catalogType() {
+ return CATALOG_TYPE;
+ }
+
+ @Override
+ public boolean canHandle(String path) {
+ // Check if the path looks like an S3 path and could be an Iceberg table
+ // A more robust implementation would check for the presence of metadata directory
+ return path != null && (path.startsWith("s3://") || path.startsWith("s3a://") || path.startsWith("s3n://"));
+ }
+
+ @Override
+ public SourceMetadata metadata(String tablePath, Map config) throws IOException {
+ S3Configuration s3Config = extractS3Config(config);
+ try {
+ IcebergTableMetadata metadata = IcebergCatalogAdapter.resolveTable(tablePath, s3Config);
+ return new IcebergSourceMetadata(metadata);
+ } catch (Exception e) {
+ throw new IOException("Failed to resolve Iceberg table metadata: " + tablePath, e);
+ }
+ }
+
+ @Override
+ public List planScan(String tablePath, Map config, List predicates) throws IOException {
+ S3Configuration s3Config = extractS3Config(config);
+ S3FileIO fileIO = null;
+
+ try {
+ // Resolve the table metadata first
+ IcebergTableMetadata metadata = IcebergCatalogAdapter.resolveTable(tablePath, s3Config);
+
+ // Create FileIO and table for scanning
+ fileIO = S3FileIOFactory.create(s3Config);
+ StaticTableOperations ops = new StaticTableOperations(metadata.metadataLocation(), fileIO);
+ Table table = new BaseTable(ops, tablePath);
+
+ // Create a table scan
+ TableScan scan = table.newScan();
+
+ // Apply predicates if any (convert from generic predicates to Iceberg expressions)
+ // For now, we don't apply predicates at the scan planning level
+ // Predicate pushdown happens during actual reading via IcebergSourceOperatorFactory
+
+ // Plan the files to read
+ List dataFiles = new ArrayList<>();
+ try (CloseableIterable fileTasks = scan.planFiles()) {
+ for (FileScanTask task : fileTasks) {
+ dataFiles.add(new IcebergDataFile(task));
+ }
+ }
+
+ return dataFiles;
+ } catch (Exception e) {
+ throw new IOException("Failed to plan Iceberg table scan: " + tablePath, e);
+ } finally {
+ IOUtils.closeWhileHandlingException(fileIO);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to close at the catalog level
+ }
+
+ /**
+ * Extract S3 configuration from the config map.
+ */
+ private S3Configuration extractS3Config(Map config) {
+ if (config == null || config.isEmpty()) {
+ return null;
+ }
+
+ String accessKey = (String) config.get("access_key");
+ String secretKey = (String) config.get("secret_key");
+ String endpoint = (String) config.get("endpoint");
+ String region = (String) config.get("region");
+
+ return S3Configuration.fromFields(accessKey, secretKey, endpoint, region);
+ }
+
+ /**
+ * Implementation of DataFile for Iceberg file scan tasks.
+ */
+ private static class IcebergDataFile implements DataFile {
+ private final FileScanTask task;
+
+ IcebergDataFile(FileScanTask task) {
+ this.task = task;
+ }
+
+ @Override
+ public String path() {
+ return task.file().path().toString();
+ }
+
+ @Override
+ public String format() {
+ return task.file().format().name().toLowerCase(java.util.Locale.ROOT);
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return task.file().fileSizeInBytes();
+ }
+
+ @Override
+ public long recordCount() {
+ return task.file().recordCount();
+ }
+
+ @Override
+ public Map partitionValues() {
+ // For now, return empty map - partition values would require schema context
+ return Collections.emptyMap();
+ }
+ }
+
+ /**
+ * Adapter that wraps IcebergTableMetadata to implement SourceMetadata.
+ */
+ private static class IcebergSourceMetadata implements SourceMetadata {
+ private final IcebergTableMetadata metadata;
+
+ IcebergSourceMetadata(IcebergTableMetadata metadata) {
+ this.metadata = metadata;
+ }
+
+ @Override
+ public List schema() {
+ return metadata.attributes();
+ }
+
+ @Override
+ public String sourceType() {
+ return metadata.sourceType();
+ }
+
+ @Override
+ public String location() {
+ return metadata.tablePath();
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadata.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadata.java
new file mode 100644
index 0000000000000..0445ed394091c
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadata.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.Check;
+import org.elasticsearch.xpack.esql.datasources.ExternalSourceMetadata;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Metadata for an Iceberg table or Parquet file.
+ * Contains schema information resolved from Iceberg/Parquet metadata.
+ */
+public class IcebergTableMetadata implements ExternalSourceMetadata {
+
+ private final String tablePath;
+ private final Schema schema;
+ private final List attributes;
+ private final S3Configuration s3Config;
+ private final String sourceType;
+ private final String metadataLocation; // For Iceberg tables, stores the metadata file location
+
+ public IcebergTableMetadata(String tablePath, Schema schema, S3Configuration s3Config, String sourceType) {
+ this(tablePath, schema, s3Config, sourceType, null);
+ }
+
+ public IcebergTableMetadata(String tablePath, Schema schema, S3Configuration s3Config, String sourceType, String metadataLocation) {
+ Check.notNull(tablePath, "tablePath must not be null");
+ Check.notNull(schema, "schema must not be null");
+ Check.notNull(sourceType, "sourceType must not be null");
+ this.tablePath = tablePath;
+ this.schema = schema;
+ this.s3Config = s3Config;
+ this.sourceType = sourceType;
+ this.metadataLocation = metadataLocation;
+ this.attributes = buildAttributes();
+ }
+
+ private List buildAttributes() {
+ List attrs = new ArrayList<>();
+ for (Types.NestedField field : schema.columns()) {
+ DataType esqlType = mapIcebergTypeToEsql(field.type());
+ // Skip unsupported types (MAP, STRUCT, etc.)
+ if (esqlType != null && esqlType != DataType.UNSUPPORTED) {
+ attrs.add(new ReferenceAttribute(Source.EMPTY, field.name(), esqlType));
+ }
+ }
+ return attrs;
+ }
+
+ /**
+ * Map Iceberg/Parquet types to ESQL DataTypes.
+ * Basic type mapping - can be extended for more complex types.
+ *
+ * For LIST types, returns the element type since ESQL handles multi-values implicitly.
+ * This allows multi-value fields in Parquet to be queried naturally in ESQL.
+ */
+ private static DataType mapIcebergTypeToEsql(Type icebergType) {
+ if (icebergType.isPrimitiveType()) {
+ return mapPrimitiveType(icebergType.asPrimitiveType());
+ }
+
+ // Handle LIST types - extract element type for multi-value fields
+ if (icebergType.typeId() == Type.TypeID.LIST) {
+ Types.ListType listType = (Types.ListType) icebergType;
+ Type elementType = listType.elementType();
+ // Recursively map the element type (handles nested lists and primitive elements)
+ return mapIcebergTypeToEsql(elementType);
+ }
+
+ // For other complex types (MAP, STRUCT), return UNSUPPORTED for now
+ return DataType.UNSUPPORTED;
+ }
+
+ /**
+ * Map Iceberg primitive types to ESQL DataTypes.
+ */
+ private static DataType mapPrimitiveType(Type.PrimitiveType primitiveType) {
+ switch (primitiveType.typeId()) {
+ case BOOLEAN:
+ return DataType.BOOLEAN;
+ case INTEGER:
+ return DataType.INTEGER;
+ case LONG:
+ return DataType.LONG;
+ case FLOAT:
+ return DataType.DOUBLE; // ESQL uses DOUBLE for float types
+ case DOUBLE:
+ return DataType.DOUBLE;
+ case STRING:
+ return DataType.KEYWORD;
+ case TIMESTAMP:
+ return DataType.DATETIME;
+ case DATE:
+ return DataType.DATETIME;
+ case BINARY:
+ case FIXED:
+ // Binary types could map to KEYWORD for now
+ return DataType.KEYWORD;
+ case DECIMAL:
+ return DataType.DOUBLE; // Simplified mapping - decimals converted to doubles
+ default:
+ return DataType.UNSUPPORTED;
+ }
+ }
+
+ @Override
+ public String tablePath() {
+ return tablePath;
+ }
+
+ @Override
+ public List attributes() {
+ return attributes;
+ }
+
+ @Override
+ public String sourceType() {
+ return sourceType;
+ }
+
+ /**
+ * Returns the Iceberg schema for this table.
+ * This is the native Iceberg schema, not the ESQL schema.
+ */
+ public Schema icebergSchema() {
+ return schema;
+ }
+
+ @Override
+ public List schema() {
+ return attributes;
+ }
+
+ @Override
+ public String location() {
+ return tablePath;
+ }
+
+ public S3Configuration s3Config() {
+ return s3Config;
+ }
+
+ public String metadataLocation() {
+ return metadataLocation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ IcebergTableMetadata that = (IcebergTableMetadata) o;
+ // Compare schema by structure (sameSchema) rather than object identity
+ return Objects.equals(tablePath, that.tablePath) && schema.sameSchema(that.schema) && Objects.equals(sourceType, that.sourceType);
+ }
+
+ @Override
+ public int hashCode() {
+ // Use schema's schemaId for hash code since sameSchema compares by structure
+ return Objects.hash(tablePath, schema.schemaId(), sourceType);
+ }
+
+ @Override
+ public String toString() {
+ return "IcebergTableMetadata{tablePath='" + tablePath + "', sourceType='" + sourceType + "', fields=" + attributes.size() + "}";
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3Configuration.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3Configuration.java
new file mode 100644
index 0000000000000..840c1f5e4858c
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3Configuration.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Configuration for S3 access, including credentials and endpoint settings.
+ * This class extracts and validates S3-related parameters from external source commands.
+ */
+public class S3Configuration {
+
+ private final String accessKey;
+ private final String secretKey;
+ private final String endpoint;
+ private final String region;
+
+ private S3Configuration(String accessKey, String secretKey, String endpoint, String region) {
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+ this.endpoint = endpoint;
+ this.region = region;
+ }
+
+ /**
+ * Parse S3 configuration from query parameters.
+ *
+ * @param params parameters from external source command
+ * @return S3Configuration instance, or null if no S3 credentials provided
+ */
+ public static S3Configuration fromParams(Map params) {
+ if (params == null || params.isEmpty()) {
+ return null;
+ }
+
+ String accessKey = extractStringParam(params, "access_key");
+ String secretKey = extractStringParam(params, "secret_key");
+ String endpoint = extractStringParam(params, "endpoint");
+ String region = extractStringParam(params, "region");
+
+ // If no credentials are provided, return null (will use default AWS credentials chain)
+ if (accessKey == null && secretKey == null && endpoint == null && region == null) {
+ return null;
+ }
+
+ return new S3Configuration(accessKey, secretKey, endpoint, region);
+ }
+
+ /**
+ * Create S3Configuration from individual fields (used for deserialization).
+ *
+ * @param accessKey access key (nullable)
+ * @param secretKey secret key (nullable)
+ * @param endpoint endpoint (nullable)
+ * @param region region (nullable)
+ * @return S3Configuration instance, or null if all fields are null
+ */
+ public static S3Configuration fromFields(String accessKey, String secretKey, String endpoint, String region) {
+ // If no fields are provided, return null (will use default AWS credentials chain)
+ if (accessKey == null && secretKey == null && endpoint == null && region == null) {
+ return null;
+ }
+ return new S3Configuration(accessKey, secretKey, endpoint, region);
+ }
+
+ private static String extractStringParam(Map params, String key) {
+ Expression expr = params.get(key);
+ if (expr instanceof org.elasticsearch.xpack.esql.core.expression.Literal literal) {
+ Object value = literal.value();
+ if (value instanceof BytesRef bytesRef) {
+ return BytesRefs.toString(bytesRef);
+ }
+ return value != null ? value.toString() : null;
+ }
+ return null;
+ }
+
+ public String accessKey() {
+ return accessKey;
+ }
+
+ public String secretKey() {
+ return secretKey;
+ }
+
+ public String endpoint() {
+ return endpoint;
+ }
+
+ public String region() {
+ return region;
+ }
+
+ public boolean hasCredentials() {
+ return accessKey != null && secretKey != null;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ S3Configuration that = (S3Configuration) o;
+ return Objects.equals(accessKey, that.accessKey)
+ && Objects.equals(secretKey, that.secretKey)
+ && Objects.equals(endpoint, that.endpoint)
+ && Objects.equals(region, that.region);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(accessKey, secretKey, endpoint, region);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3FileIOFactory.java b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3FileIOFactory.java
new file mode 100644
index 0000000000000..c980d27b21e3e
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3FileIOFactory.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.profiles.ProfileFile;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+
+import org.apache.iceberg.aws.s3.S3FileIO;
+import org.apache.iceberg.util.SerializableSupplier;
+
+import java.net.URI;
+
+/**
+ * Factory for creating configured S3FileIO instances.
+ *
+ * This class provides a way to create Iceberg's S3FileIO without using Hadoop,
+ * replacing the previous HadoopCatalog-based approach. S3FileIO uses the AWS SDK
+ * directly and works with both real S3 endpoints and test fixtures like S3HttpFixture.
+ */
+public final class S3FileIOFactory {
+
+ // S3FileIO property keys
+ private static final String S3_ACCESS_KEY_ID = "s3.access-key-id";
+ private static final String S3_SECRET_ACCESS_KEY = "s3.secret-access-key";
+ private static final String S3_ENDPOINT = "s3.endpoint";
+ private static final String CLIENT_REGION = "client.region";
+ private static final String S3_PATH_STYLE_ACCESS = "s3.path-style-access";
+
+ private S3FileIOFactory() {
+ // Utility class - no instantiation
+ }
+
+ /**
+ * Create and configure an S3FileIO instance with the given S3 configuration.
+ *
+ * The returned S3FileIO is configured for:
+ *
+ * Static credentials if provided (access key and secret key)
+ * Custom endpoint if provided (for testing with S3-compatible services)
+ * Region if provided
+ * Path-style access (required for MinIO, LocalStack, and S3HttpFixture)
+ *
+ *
+ * @param s3Config S3 configuration (nullable - if null, uses default AWS credentials chain)
+ * @return configured S3FileIO instance (caller should close when done)
+ */
+ public static S3FileIO create(S3Configuration s3Config) {
+ // Create a pre-configured S3 client supplier
+ // This bypasses Iceberg's HTTP client configuration which uses package-private classes
+ // that can't be accessed via reflection in Elasticsearch's classloader environment
+ SerializableSupplier s3ClientSupplier = (SerializableSupplier & java.io.Serializable) () -> {
+ S3ClientBuilder builder = S3Client.builder();
+
+ // Always set a region to avoid auto-detection issues
+ Region region = Region.US_EAST_1; // Default region
+
+ // CRITICAL: Create an empty profile file to prevent AWS SDK from reading ~/.aws/credentials
+ // and ~/.aws/config files, which would trigger Elasticsearch entitlement violations.
+ // We must set BOTH the profile file AND the profile file supplier to empty values.
+ ProfileFile emptyProfileFile = ProfileFile.builder()
+ .type(ProfileFile.Type.CREDENTIALS)
+ .content(new java.io.ByteArrayInputStream(new byte[0]))
+ .build();
+
+ // Use a supplier that returns the empty profile file to prevent lazy loading of default files
+ java.util.function.Supplier emptyProfileSupplier = () -> emptyProfileFile;
+
+ builder.overrideConfiguration(c -> {
+ c.defaultProfileFile(emptyProfileFile);
+ c.defaultProfileFileSupplier(emptyProfileSupplier);
+ });
+
+ // Always provide explicit credentials
+ if (s3Config != null && s3Config.hasCredentials()) {
+ AwsBasicCredentials credentials = AwsBasicCredentials.create(s3Config.accessKey(), s3Config.secretKey());
+ builder.credentialsProvider(StaticCredentialsProvider.create(credentials));
+ } else {
+ // Use default test credentials that match the S3 fixture expectations
+ // These match the credentials in S3FixtureUtils
+ AwsBasicCredentials testCredentials = AwsBasicCredentials.create("test-access-key", "test-secret-key");
+ builder.credentialsProvider(StaticCredentialsProvider.create(testCredentials));
+ }
+
+ if (s3Config != null) {
+ if (s3Config.endpoint() != null) {
+ builder.endpointOverride(URI.create(s3Config.endpoint()));
+ }
+ if (s3Config.region() != null) {
+ region = Region.of(s3Config.region());
+ }
+ }
+
+ builder.region(region);
+
+ // Enable path-style access for compatibility with MinIO, LocalStack, and S3HttpFixture
+ builder.forcePathStyle(true);
+
+ // Use URL connection HTTP client to avoid entitlement issues
+ // The Apache HTTP client creates daemon threads which are blocked by Elasticsearch's entitlement system
+ builder.httpClient(UrlConnectionHttpClient.builder().build());
+
+ return builder.build();
+ };
+
+ // Initialize S3FileIO with the pre-configured S3 client
+ return new S3FileIO(s3ClientSupplier);
+ }
+
+ /**
+ * Create and configure an S3FileIO instance from individual configuration values.
+ *
+ * This is a convenience method for cases where the configuration values are
+ * available directly rather than through an S3Configuration object.
+ *
+ * @param accessKey S3 access key (nullable)
+ * @param secretKey S3 secret key (nullable)
+ * @param endpoint S3 endpoint URL (nullable)
+ * @param region AWS region (nullable)
+ * @return configured S3FileIO instance (caller should close when done)
+ */
+ public static S3FileIO create(String accessKey, String secretKey, String endpoint, String region) {
+ S3Configuration s3Config = S3Configuration.fromFields(accessKey, secretKey, endpoint, region);
+ return create(s3Config);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/esql-datasource-iceberg/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..394e5e38d9f59
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,3 @@
+ALL-UNNAMED:
+ - manage_threads
+ - outbound_network
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin b/x-pack/plugin/esql-datasource-iceberg/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
new file mode 100644
index 0000000000000..a20e46e833911
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
@@ -0,0 +1 @@
+org.elasticsearch.xpack.esql.datasource.iceberg.IcebergDataSourcePlugin
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapterTests.java b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapterTests.java
new file mode 100644
index 0000000000000..e817873365679
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergCatalogAdapterTests.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.elasticsearch.test.ESTestCase;
+
+/**
+ * Unit tests for IcebergCatalogAdapter.
+ * Tests the version number extraction logic used for finding metadata files.
+ *
+ * Note: The main resolveTable() and findLatestMetadataFile() methods require
+ * actual S3 connectivity and are tested via integration tests.
+ */
+public class IcebergCatalogAdapterTests extends ESTestCase {
+
+ public void testExtractVersionNumberFromSimplePath() throws Exception {
+ int version = invokeExtractVersionNumber("v1.metadata.json");
+ assertEquals(1, version);
+ }
+
+ public void testExtractVersionNumberFromFullPath() throws Exception {
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/v42.metadata.json");
+ assertEquals(42, version);
+ }
+
+ public void testExtractVersionNumberFromLargeVersion() throws Exception {
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/v9999.metadata.json");
+ assertEquals(9999, version);
+ }
+
+ public void testExtractVersionNumberFromPathWithNestedDirs() throws Exception {
+ int version = invokeExtractVersionNumber("s3://bucket/path/to/table/metadata/v123.metadata.json");
+ assertEquals(123, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForInvalidFormat() throws Exception {
+ // Missing v prefix
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/1.metadata.json");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForWrongExtension() throws Exception {
+ // Wrong file extension
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/v1.json");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForNonNumeric() throws Exception {
+ // Non-numeric version
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/vABC.metadata.json");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForEmptyFilename() throws Exception {
+ int version = invokeExtractVersionNumber("");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForJustExtension() throws Exception {
+ int version = invokeExtractVersionNumber(".metadata.json");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForSnapshotFile() throws Exception {
+ // Iceberg snapshot files have different naming
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/snap-123456789.avro");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberReturnsZeroForVersionHintFile() throws Exception {
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/version-hint.text");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberWithTrailingSlash() throws Exception {
+ // Edge case: path ending with slash (shouldn't happen but handle gracefully)
+ int version = invokeExtractVersionNumber("s3://bucket/table/metadata/");
+ assertEquals(0, version);
+ }
+
+ public void testExtractVersionNumberFromLocalPath() throws Exception {
+ // Local filesystem path format
+ int version = invokeExtractVersionNumber("/path/to/table/metadata/v7.metadata.json");
+ assertEquals(7, version);
+ }
+
+ public void testExtractVersionNumberFromWindowsPath() throws Exception {
+ // Windows-style path (forward slashes work)
+ int version = invokeExtractVersionNumber("C:/data/table/metadata/v15.metadata.json");
+ assertEquals(15, version);
+ }
+
+ public void testMetadataDirectorySuffix() {
+ // Verify the expected metadata directory structure
+ String tablePath = "s3://bucket/table";
+ String expectedMetadataPath = tablePath + "/metadata/v1.metadata.json";
+ assertTrue(expectedMetadataPath.endsWith(".metadata.json"));
+ assertTrue(expectedMetadataPath.contains("/metadata/"));
+ }
+
+ public void testSourceTypeConstant() {
+ // The source type should be "iceberg"
+ // This validates that any IcebergTableMetadata returned will have the correct sourceType
+ String expectedSourceType = "iceberg";
+
+ // We can verify this by checking that IcebergTableMetadata created with "iceberg" works
+ org.apache.iceberg.Schema schema = new org.apache.iceberg.Schema(
+ org.apache.iceberg.types.Types.NestedField.required(1, "id", org.apache.iceberg.types.Types.LongType.get())
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+ assertEquals(expectedSourceType, metadata.sourceType());
+ }
+
+ private int invokeExtractVersionNumber(String path) {
+ return IcebergCatalogAdapter.extractVersionNumber(path);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFiltersTests.java b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFiltersTests.java
new file mode 100644
index 0000000000000..4ca23cfaf33c5
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergPushdownFiltersTests.java
@@ -0,0 +1,394 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.iceberg.expressions.Expression;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.predicate.Range;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
+import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNull;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+
+import java.time.ZoneOffset;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.type.EsField.TimeSeriesFieldType;
+
+/**
+ * Unit tests for IcebergPushdownFilters.
+ * Tests conversion of ESQL expressions to Iceberg filter expressions.
+ */
+public class IcebergPushdownFiltersTests extends ESTestCase {
+
+ private static final Source SOURCE = Source.EMPTY;
+
+ public void testEqualsStringField() {
+ FieldAttribute field = createField("name", DataType.KEYWORD);
+ Literal value = literal("Alice");
+
+ Equals equals = new Equals(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(equals);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'name' in: " + resultStr, resultStr.contains("name"));
+ assertTrue("Expected value 'Alice' in: " + resultStr, resultStr.contains("Alice"));
+ }
+
+ public void testEqualsIntegerField() {
+ FieldAttribute field = createField("age", DataType.INTEGER);
+ Literal value = literal(25);
+
+ Equals equals = new Equals(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(equals);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ // Value is converted to string representation
+ assertTrue("Expected field 'age' in: " + resultStr, resultStr.contains("age"));
+ assertTrue("Expected value '25' in: " + resultStr, resultStr.contains("25"));
+ }
+
+ public void testNotEquals() {
+ FieldAttribute field = createField("status", DataType.KEYWORD);
+ Literal value = literal("inactive");
+
+ NotEquals notEquals = new NotEquals(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(notEquals);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'status' in: " + resultStr, resultStr.contains("status"));
+ assertTrue("Expected value 'inactive' in: " + resultStr, resultStr.contains("inactive"));
+ }
+
+ public void testLessThan() {
+ FieldAttribute field = createField("price", DataType.DOUBLE);
+ Literal value = literal(100.0);
+
+ LessThan lessThan = new LessThan(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(lessThan);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'price' in: " + resultStr, resultStr.contains("price"));
+ assertTrue("Expected value '100.0' in: " + resultStr, resultStr.contains("100.0"));
+ }
+
+ public void testLessThanOrEqual() {
+ FieldAttribute field = createField("quantity", DataType.INTEGER);
+ Literal value = literal(10);
+
+ LessThanOrEqual lessThanOrEqual = new LessThanOrEqual(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(lessThanOrEqual);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'quantity' in: " + resultStr, resultStr.contains("quantity"));
+ assertTrue("Expected value '10' in: " + resultStr, resultStr.contains("10"));
+ }
+
+ public void testGreaterThan() {
+ FieldAttribute field = createField("score", DataType.DOUBLE);
+ Literal value = literal(90.0);
+
+ GreaterThan greaterThan = new GreaterThan(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(greaterThan);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'score' in: " + resultStr, resultStr.contains("score"));
+ assertTrue("Expected value '90.0' in: " + resultStr, resultStr.contains("90.0"));
+ }
+
+ public void testGreaterThanOrEqual() {
+ FieldAttribute field = createField("level", DataType.INTEGER);
+ Literal value = literal(5);
+
+ GreaterThanOrEqual greaterThanOrEqual = new GreaterThanOrEqual(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(greaterThanOrEqual);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'level' in: " + resultStr, resultStr.contains("level"));
+ assertTrue("Expected value '5' in: " + resultStr, resultStr.contains("5"));
+ }
+
+ public void testIsNull() {
+ FieldAttribute field = createField("email", DataType.KEYWORD);
+
+ IsNull isNull = new IsNull(SOURCE, field);
+ Expression result = IcebergPushdownFilters.convert(isNull);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected is_null in: " + resultStr, resultStr.contains("is_null"));
+ assertTrue("Expected field 'email' in: " + resultStr, resultStr.contains("email"));
+ }
+
+ public void testIsNotNull() {
+ FieldAttribute field = createField("email", DataType.KEYWORD);
+
+ IsNotNull isNotNull = new IsNotNull(SOURCE, field);
+ Expression result = IcebergPushdownFilters.convert(isNotNull);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected not_null in: " + resultStr, resultStr.contains("not_null"));
+ assertTrue("Expected field 'email' in: " + resultStr, resultStr.contains("email"));
+ }
+
+ public void testIn() {
+ FieldAttribute field = createField("category", DataType.KEYWORD);
+ List values = List.of(literal("A"), literal("B"), literal("C"));
+
+ In inExpr = new In(SOURCE, field, values);
+ Expression result = IcebergPushdownFilters.convert(inExpr);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'category' in: " + resultStr, resultStr.contains("category"));
+ assertTrue("Expected 'in' operator in: " + resultStr, resultStr.contains("in"));
+ assertTrue("Expected value 'A' in: " + resultStr, resultStr.contains("A"));
+ assertTrue("Expected value 'B' in: " + resultStr, resultStr.contains("B"));
+ assertTrue("Expected value 'C' in: " + resultStr, resultStr.contains("C"));
+ }
+
+ public void testRangeInclusiveBoth() {
+ FieldAttribute field = createField("value", DataType.INTEGER);
+ Literal lower = literal(10);
+ Literal upper = literal(20);
+
+ Range range = new Range(SOURCE, field, lower, true, upper, true, ZoneOffset.UTC);
+ Expression result = IcebergPushdownFilters.convert(range);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'value' in: " + resultStr, resultStr.contains("value"));
+ assertTrue("Expected value '10' in: " + resultStr, resultStr.contains("10"));
+ assertTrue("Expected value '20' in: " + resultStr, resultStr.contains("20"));
+ assertTrue("Expected 'and' operator in: " + resultStr, resultStr.toLowerCase(java.util.Locale.ROOT).contains("and"));
+ }
+
+ public void testRangeExclusiveBoth() {
+ FieldAttribute field = createField("value", DataType.INTEGER);
+ Literal lower = literal(10);
+ Literal upper = literal(20);
+
+ Range range = new Range(SOURCE, field, lower, false, upper, false, ZoneOffset.UTC);
+ Expression result = IcebergPushdownFilters.convert(range);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'value' in: " + resultStr, resultStr.contains("value"));
+ assertTrue("Expected value '10' in: " + resultStr, resultStr.contains("10"));
+ assertTrue("Expected value '20' in: " + resultStr, resultStr.contains("20"));
+ assertTrue("Expected 'and' operator in: " + resultStr, resultStr.toLowerCase(java.util.Locale.ROOT).contains("and"));
+ }
+
+ public void testAndExpression() {
+ FieldAttribute field1 = createField("status", DataType.KEYWORD);
+ FieldAttribute field2 = createField("active", DataType.BOOLEAN);
+ Literal value1 = literal("approved");
+ Literal value2 = literal(true);
+
+ Equals equals1 = new Equals(SOURCE, field1, value1);
+ Equals equals2 = new Equals(SOURCE, field2, value2);
+ And and = new And(SOURCE, equals1, equals2);
+
+ Expression result = IcebergPushdownFilters.convert(and);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'status' in: " + resultStr, resultStr.contains("status"));
+ assertTrue("Expected value 'approved' in: " + resultStr, resultStr.contains("approved"));
+ assertTrue("Expected field 'active' in: " + resultStr, resultStr.contains("active"));
+ assertTrue("Expected value 'true' in: " + resultStr, resultStr.contains("true"));
+ assertTrue("Expected 'and' operator in: " + resultStr, resultStr.toLowerCase(java.util.Locale.ROOT).contains("and"));
+ }
+
+ public void testOrExpression() {
+ FieldAttribute field = createField("category", DataType.KEYWORD);
+ Literal value1 = literal("A");
+ Literal value2 = literal("B");
+
+ Equals equals1 = new Equals(SOURCE, field, value1);
+ Equals equals2 = new Equals(SOURCE, field, value2);
+ Or or = new Or(SOURCE, equals1, equals2);
+
+ Expression result = IcebergPushdownFilters.convert(or);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'category' in: " + resultStr, resultStr.contains("category"));
+ assertTrue("Expected value 'A' in: " + resultStr, resultStr.contains("A"));
+ assertTrue("Expected value 'B' in: " + resultStr, resultStr.contains("B"));
+ assertTrue("Expected 'or' operator in: " + resultStr, resultStr.toLowerCase(java.util.Locale.ROOT).contains("or"));
+ }
+
+ public void testNotExpression() {
+ FieldAttribute field = createField("status", DataType.KEYWORD);
+ Literal value = literal("inactive");
+
+ Equals equals = new Equals(SOURCE, field, value);
+ Not not = new Not(SOURCE, equals);
+
+ Expression result = IcebergPushdownFilters.convert(not);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected 'not' operator in: " + resultStr, resultStr.toLowerCase(java.util.Locale.ROOT).contains("not"));
+ assertTrue("Expected field 'status' in: " + resultStr, resultStr.contains("status"));
+ assertTrue("Expected value 'inactive' in: " + resultStr, resultStr.contains("inactive"));
+ }
+
+ public void testNestedAndOrExpression() {
+ FieldAttribute field1 = createField("status", DataType.KEYWORD);
+ FieldAttribute field2 = createField("priority", DataType.INTEGER);
+ FieldAttribute field3 = createField("category", DataType.KEYWORD);
+
+ Equals statusActive = new Equals(SOURCE, field1, literal("active"));
+ GreaterThan highPriority = new GreaterThan(SOURCE, field2, literal(5));
+ Equals categoryA = new Equals(SOURCE, field3, literal("A"));
+
+ And andExpr = new And(SOURCE, statusActive, highPriority);
+ Or orExpr = new Or(SOURCE, andExpr, categoryA);
+
+ Expression result = IcebergPushdownFilters.convert(orExpr);
+
+ assertNotNull(result);
+ String resultStr = result.toString();
+ assertTrue("Expected field 'status' in: " + resultStr, resultStr.contains("status"));
+ assertTrue("Expected value 'active' in: " + resultStr, resultStr.contains("active"));
+ assertTrue("Expected field 'priority' in: " + resultStr, resultStr.contains("priority"));
+ assertTrue("Expected value '5' in: " + resultStr, resultStr.contains("5"));
+ assertTrue("Expected field 'category' in: " + resultStr, resultStr.contains("category"));
+ assertTrue("Expected value 'A' in: " + resultStr, resultStr.contains("A"));
+ }
+
+ public void testNullForUnsupportedExpression() {
+ // A literal by itself should return null (not a supported predicate)
+ Literal literal = literal("value");
+ Expression result = IcebergPushdownFilters.convert(literal);
+
+ assertNull(result);
+ }
+
+ public void testNullForAndWithUnsupportedChild() {
+ FieldAttribute field = createField("status", DataType.KEYWORD);
+ Equals equals = new Equals(SOURCE, field, literal("active"));
+ Literal unsupported = literal("value");
+
+ And and = new And(SOURCE, equals, unsupported);
+ Expression result = IcebergPushdownFilters.convert(and);
+
+ // Should return null because one child is unsupported
+ assertNull(result);
+ }
+
+ public void testNullForOrWithUnsupportedChild() {
+ FieldAttribute field = createField("status", DataType.KEYWORD);
+ Equals equals = new Equals(SOURCE, field, literal("active"));
+ Literal unsupported = literal("value");
+
+ Or or = new Or(SOURCE, equals, unsupported);
+ Expression result = IcebergPushdownFilters.convert(or);
+
+ // Should return null because one child is unsupported
+ assertNull(result);
+ }
+
+ public void testNullForNotWithUnsupportedChild() {
+ Literal unsupported = literal("value");
+ Not not = new Not(SOURCE, unsupported);
+
+ Expression result = IcebergPushdownFilters.convert(not);
+
+ // Should return null because child is unsupported
+ assertNull(result);
+ }
+
+ public void testInWithNonFoldableValue() {
+ FieldAttribute field = createField("category", DataType.KEYWORD);
+ FieldAttribute nonFoldable = createField("other", DataType.KEYWORD);
+ List values = List.of(
+ literal("A"),
+ nonFoldable // Not foldable
+ );
+
+ In inExpr = new In(SOURCE, field, values);
+ Expression result = IcebergPushdownFilters.convert(inExpr);
+
+ // Should return null because not all values are foldable
+ assertNull(result);
+ }
+
+ public void testEqualsWithNonFoldableValue() {
+ FieldAttribute field1 = createField("name", DataType.KEYWORD);
+ FieldAttribute field2 = createField("alias", DataType.KEYWORD);
+
+ // field = another_field (not a literal)
+ Equals equals = new Equals(SOURCE, field1, field2);
+ Expression result = IcebergPushdownFilters.convert(equals);
+
+ // Should return null because right side is not foldable
+ assertNull(result);
+ }
+
+ public void testBytesRefValueConversion() {
+ FieldAttribute field = createField("name", DataType.KEYWORD);
+ Literal value = new Literal(SOURCE, new BytesRef("test_value"), DataType.KEYWORD);
+
+ Equals equals = new Equals(SOURCE, field, value);
+ Expression result = IcebergPushdownFilters.convert(equals);
+
+ assertNotNull(result);
+ // BytesRef should be converted to string
+ assertTrue(result.toString().contains("test_value"));
+ }
+
+ private FieldAttribute createField(String name, DataType dataType) {
+ return new FieldAttribute(SOURCE, name, new EsField(name, dataType, Collections.emptyMap(), true, TimeSeriesFieldType.NONE));
+ }
+
+ private Literal literal(Object value) {
+ DataType dataType;
+ Object literalValue = value;
+ if (value instanceof String s) {
+ dataType = DataType.KEYWORD;
+ literalValue = new BytesRef(s);
+ } else if (value instanceof Integer) {
+ dataType = DataType.INTEGER;
+ } else if (value instanceof Long) {
+ dataType = DataType.LONG;
+ } else if (value instanceof Double) {
+ dataType = DataType.DOUBLE;
+ } else if (value instanceof Boolean) {
+ dataType = DataType.BOOLEAN;
+ } else {
+ dataType = DataType.KEYWORD;
+ }
+ return new Literal(SOURCE, literalValue, dataType);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadataTests.java b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadataTests.java
new file mode 100644
index 0000000000000..077055e88d255
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/IcebergTableMetadataTests.java
@@ -0,0 +1,296 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.Types;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+
+/**
+ * Unit tests for IcebergTableMetadata.
+ * Tests schema conversion from Iceberg types to ESQL DataTypes and metadata accessors.
+ */
+public class IcebergTableMetadataTests extends ESTestCase {
+
+ public void testBooleanTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "active", Types.BooleanType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("active", attributes.get(0).name());
+ assertEquals(DataType.BOOLEAN, attributes.get(0).dataType());
+ }
+
+ public void testIntegerTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "count", Types.IntegerType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("count", attributes.get(0).name());
+ assertEquals(DataType.INTEGER, attributes.get(0).dataType());
+ }
+
+ public void testLongTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("id", attributes.get(0).name());
+ assertEquals(DataType.LONG, attributes.get(0).dataType());
+ }
+
+ public void testFloatTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "temperature", Types.FloatType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("temperature", attributes.get(0).name());
+ assertEquals(DataType.DOUBLE, attributes.get(0).dataType()); // Float maps to DOUBLE
+ }
+
+ public void testDoubleTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "score", Types.DoubleType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("score", attributes.get(0).name());
+ assertEquals(DataType.DOUBLE, attributes.get(0).dataType());
+ }
+
+ public void testStringTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "name", Types.StringType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("name", attributes.get(0).name());
+ assertEquals(DataType.KEYWORD, attributes.get(0).dataType());
+ }
+
+ public void testTimestampTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "created_at", Types.TimestampType.withoutZone()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("created_at", attributes.get(0).name());
+ assertEquals(DataType.DATETIME, attributes.get(0).dataType());
+ }
+
+ public void testDateTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "birth_date", Types.DateType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("birth_date", attributes.get(0).name());
+ assertEquals(DataType.DATETIME, attributes.get(0).dataType());
+ }
+
+ public void testBinaryTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "data", Types.BinaryType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("data", attributes.get(0).name());
+ assertEquals(DataType.KEYWORD, attributes.get(0).dataType());
+ }
+
+ public void testDecimalTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "price", Types.DecimalType.of(10, 2)));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("price", attributes.get(0).name());
+ assertEquals(DataType.DOUBLE, attributes.get(0).dataType()); // Decimal maps to DOUBLE
+ }
+
+ public void testListTypeMapping() {
+ // List of integers - should map to INTEGER (element type)
+ Schema schema = new Schema(Types.NestedField.required(1, "scores", Types.ListType.ofRequired(2, Types.IntegerType.get())));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("scores", attributes.get(0).name());
+ assertEquals(DataType.INTEGER, attributes.get(0).dataType()); // Element type
+ }
+
+ public void testListOfStringsTypeMapping() {
+ Schema schema = new Schema(Types.NestedField.required(1, "tags", Types.ListType.ofRequired(2, Types.StringType.get())));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(1, attributes.size());
+ assertEquals("tags", attributes.get(0).name());
+ assertEquals(DataType.KEYWORD, attributes.get(0).dataType());
+ }
+
+ public void testMapTypeReturnsUnsupported() {
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "properties", Types.MapType.ofRequired(2, 3, Types.StringType.get(), Types.StringType.get()))
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ // Maps return UNSUPPORTED, so no attributes are added
+ List attributes = metadata.attributes();
+ assertEquals(0, attributes.size());
+ }
+
+ public void testStructTypeReturnsUnsupported() {
+ Schema schema = new Schema(
+ Types.NestedField.required(
+ 1,
+ "address",
+ Types.StructType.of(
+ Types.NestedField.required(2, "street", Types.StringType.get()),
+ Types.NestedField.required(3, "city", Types.StringType.get())
+ )
+ )
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ // Structs return UNSUPPORTED, so no attributes are added
+ List attributes = metadata.attributes();
+ assertEquals(0, attributes.size());
+ }
+
+ public void testMultipleColumns() {
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required(2, "name", Types.StringType.get()),
+ Types.NestedField.required(3, "active", Types.BooleanType.get()),
+ Types.NestedField.required(4, "score", Types.DoubleType.get())
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ List attributes = metadata.attributes();
+ assertEquals(4, attributes.size());
+
+ assertEquals("id", attributes.get(0).name());
+ assertEquals(DataType.LONG, attributes.get(0).dataType());
+
+ assertEquals("name", attributes.get(1).name());
+ assertEquals(DataType.KEYWORD, attributes.get(1).dataType());
+
+ assertEquals("active", attributes.get(2).name());
+ assertEquals(DataType.BOOLEAN, attributes.get(2).dataType());
+
+ assertEquals("score", attributes.get(3).name());
+ assertEquals(DataType.DOUBLE, attributes.get(3).dataType());
+ }
+
+ public void testTablePathAccessor() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ String tablePath = "s3://my-bucket/my-table";
+ IcebergTableMetadata metadata = new IcebergTableMetadata(tablePath, schema, null, "iceberg");
+
+ assertEquals(tablePath, metadata.tablePath());
+ assertEquals(tablePath, metadata.location());
+ }
+
+ public void testSourceTypeAccessor() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ assertEquals("iceberg", metadata.sourceType());
+ }
+
+ public void testIcebergSchemaAccessor() {
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required(2, "name", Types.StringType.get())
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ assertSame(schema, metadata.icebergSchema());
+ }
+
+ public void testSchemaAccessor() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ assertSame(metadata.attributes(), metadata.schema());
+ }
+
+ public void testS3ConfigAccessor() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ S3Configuration s3Config = S3Configuration.fromFields("accessKey", "secretKey", "endpoint", "us-east-1");
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, s3Config, "iceberg");
+
+ assertSame(s3Config, metadata.s3Config());
+ }
+
+ public void testMetadataLocationAccessor() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ String metadataLocation = "s3://bucket/table/metadata/v1.metadata.json";
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg", metadataLocation);
+
+ assertEquals(metadataLocation, metadata.metadataLocation());
+ }
+
+ public void testMetadataLocationNullByDefault() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ assertNull(metadata.metadataLocation());
+ }
+
+ public void testEqualsAndHashCode() {
+ Schema schema1 = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ Schema schema2 = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+
+ IcebergTableMetadata metadata1 = new IcebergTableMetadata("s3://bucket/table", schema1, null, "iceberg");
+ IcebergTableMetadata metadata2 = new IcebergTableMetadata("s3://bucket/table", schema2, null, "iceberg");
+
+ assertEquals(metadata1, metadata2);
+ assertEquals(metadata1.hashCode(), metadata2.hashCode());
+ }
+
+ public void testNotEqualsDifferentPath() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+
+ IcebergTableMetadata metadata1 = new IcebergTableMetadata("s3://bucket/table1", schema, null, "iceberg");
+ IcebergTableMetadata metadata2 = new IcebergTableMetadata("s3://bucket/table2", schema, null, "iceberg");
+
+ assertNotEquals(metadata1, metadata2);
+ }
+
+ public void testNotEqualsDifferentSourceType() {
+ Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+
+ IcebergTableMetadata metadata1 = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+ IcebergTableMetadata metadata2 = new IcebergTableMetadata("s3://bucket/table", schema, null, "parquet");
+
+ assertNotEquals(metadata1, metadata2);
+ }
+
+ public void testToString() {
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required(2, "name", Types.StringType.get())
+ );
+ IcebergTableMetadata metadata = new IcebergTableMetadata("s3://bucket/table", schema, null, "iceberg");
+
+ String toString = metadata.toString();
+ assertTrue(toString.contains("s3://bucket/table"));
+ assertTrue(toString.contains("iceberg"));
+ assertTrue(toString.contains("2")); // fields count
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3ConfigurationTests.java b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3ConfigurationTests.java
new file mode 100644
index 0000000000000..b8ef8d2652263
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-iceberg/src/test/java/org/elasticsearch/xpack/esql/datasource/iceberg/S3ConfigurationTests.java
@@ -0,0 +1,272 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.iceberg;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for S3Configuration.
+ * Tests parsing S3 credentials and configuration from query parameters.
+ */
+public class S3ConfigurationTests extends ESTestCase {
+
+ private static final Source SOURCE = Source.EMPTY;
+
+ public void testFromParamsWithAllFields() {
+ Map params = new HashMap<>();
+ params.put("access_key", literal("AKIAIOSFODNN7EXAMPLE"));
+ params.put("secret_key", literal("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"));
+ params.put("endpoint", literal("http://localhost:9000"));
+ params.put("region", literal("us-east-1"));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", config.secretKey());
+ assertEquals("http://localhost:9000", config.endpoint());
+ assertEquals("us-east-1", config.region());
+ assertTrue(config.hasCredentials());
+ }
+
+ public void testFromParamsWithCredentialsOnly() {
+ Map params = new HashMap<>();
+ params.put("access_key", literal("AKIAIOSFODNN7EXAMPLE"));
+ params.put("secret_key", literal("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", config.secretKey());
+ assertNull(config.endpoint());
+ assertNull(config.region());
+ assertTrue(config.hasCredentials());
+ }
+
+ public void testFromParamsWithEndpointOnly() {
+ Map params = new HashMap<>();
+ params.put("endpoint", literal("http://localhost:9000"));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertNull(config.accessKey());
+ assertNull(config.secretKey());
+ assertEquals("http://localhost:9000", config.endpoint());
+ assertNull(config.region());
+ assertFalse(config.hasCredentials()); // No access/secret keys
+ }
+
+ public void testFromParamsWithRegionOnly() {
+ Map params = new HashMap<>();
+ params.put("region", literal("eu-west-1"));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertNull(config.accessKey());
+ assertNull(config.secretKey());
+ assertNull(config.endpoint());
+ assertEquals("eu-west-1", config.region());
+ assertFalse(config.hasCredentials());
+ }
+
+ public void testFromParamsWithNullMapReturnsNull() {
+ S3Configuration config = S3Configuration.fromParams(null);
+ assertNull(config);
+ }
+
+ public void testFromParamsWithEmptyMapReturnsNull() {
+ S3Configuration config = S3Configuration.fromParams(new HashMap<>());
+ assertNull(config);
+ }
+
+ public void testFromParamsWithNoS3ParamsReturnsNull() {
+ Map params = new HashMap<>();
+ params.put("other_param", literal("value"));
+ params.put("another_param", literal(123));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ // No S3 params present, should return null
+ assertNull(config);
+ }
+
+ public void testFromParamsWithBytesRefValue() {
+ Map params = new HashMap<>();
+ params.put("access_key", new Literal(SOURCE, new BytesRef("AKIAIOSFODNN7EXAMPLE"), DataType.KEYWORD));
+ params.put("secret_key", new Literal(SOURCE, new BytesRef("secret"), DataType.KEYWORD));
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertEquals("secret", config.secretKey());
+ }
+
+ public void testFromParamsWithPartialCredentials() {
+ Map params = new HashMap<>();
+ params.put("access_key", literal("AKIAIOSFODNN7EXAMPLE"));
+ // No secret_key
+
+ S3Configuration config = S3Configuration.fromParams(params);
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertNull(config.secretKey());
+ assertFalse(config.hasCredentials()); // Missing secret key
+ }
+
+ public void testFromFieldsWithAllFields() {
+ S3Configuration config = S3Configuration.fromFields(
+ "AKIAIOSFODNN7EXAMPLE",
+ "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ "http://localhost:9000",
+ "us-east-1"
+ );
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", config.secretKey());
+ assertEquals("http://localhost:9000", config.endpoint());
+ assertEquals("us-east-1", config.region());
+ assertTrue(config.hasCredentials());
+ }
+
+ public void testFromFieldsWithNullAccessKey() {
+ S3Configuration config = S3Configuration.fromFields(null, "secret", "http://localhost:9000", "us-east-1");
+
+ assertNotNull(config);
+ assertNull(config.accessKey());
+ assertEquals("secret", config.secretKey());
+ assertFalse(config.hasCredentials()); // Missing access key
+ }
+
+ public void testFromFieldsWithNullSecretKey() {
+ S3Configuration config = S3Configuration.fromFields("AKIAIOSFODNN7EXAMPLE", null, "http://localhost:9000", "us-east-1");
+
+ assertNotNull(config);
+ assertEquals("AKIAIOSFODNN7EXAMPLE", config.accessKey());
+ assertNull(config.secretKey());
+ assertFalse(config.hasCredentials()); // Missing secret key
+ }
+
+ public void testFromFieldsWithAllNullReturnsNull() {
+ S3Configuration config = S3Configuration.fromFields(null, null, null, null);
+ assertNull(config);
+ }
+
+ public void testHasCredentialsWithBothKeys() {
+ S3Configuration config = S3Configuration.fromFields("access", "secret", null, null);
+
+ assertTrue(config.hasCredentials());
+ }
+
+ public void testHasCredentialsWithAccessKeyOnly() {
+ S3Configuration config = S3Configuration.fromFields("access", null, "endpoint", null);
+
+ assertFalse(config.hasCredentials());
+ }
+
+ public void testHasCredentialsWithSecretKeyOnly() {
+ S3Configuration config = S3Configuration.fromFields(null, "secret", "endpoint", null);
+
+ assertFalse(config.hasCredentials());
+ }
+
+ public void testEqualsAndHashCodeSameValues() {
+ S3Configuration config1 = S3Configuration.fromFields("access", "secret", "endpoint", "region");
+ S3Configuration config2 = S3Configuration.fromFields("access", "secret", "endpoint", "region");
+
+ assertEquals(config1, config2);
+ assertEquals(config1.hashCode(), config2.hashCode());
+ }
+
+ public void testEqualsAndHashCodeDifferentAccessKey() {
+ S3Configuration config1 = S3Configuration.fromFields("access1", "secret", "endpoint", "region");
+ S3Configuration config2 = S3Configuration.fromFields("access2", "secret", "endpoint", "region");
+
+ assertNotEquals(config1, config2);
+ }
+
+ public void testEqualsAndHashCodeDifferentSecretKey() {
+ S3Configuration config1 = S3Configuration.fromFields("access", "secret1", "endpoint", "region");
+ S3Configuration config2 = S3Configuration.fromFields("access", "secret2", "endpoint", "region");
+
+ assertNotEquals(config1, config2);
+ }
+
+ public void testEqualsAndHashCodeDifferentEndpoint() {
+ S3Configuration config1 = S3Configuration.fromFields("access", "secret", "endpoint1", "region");
+ S3Configuration config2 = S3Configuration.fromFields("access", "secret", "endpoint2", "region");
+
+ assertNotEquals(config1, config2);
+ }
+
+ public void testEqualsAndHashCodeDifferentRegion() {
+ S3Configuration config1 = S3Configuration.fromFields("access", "secret", "endpoint", "region1");
+ S3Configuration config2 = S3Configuration.fromFields("access", "secret", "endpoint", "region2");
+
+ assertNotEquals(config1, config2);
+ }
+
+ public void testEqualsWithNull() {
+ S3Configuration config = S3Configuration.fromFields("access", "secret", "endpoint", "region");
+
+ assertNotEquals(null, config);
+ }
+
+ public void testEqualsWithDifferentClass() {
+ S3Configuration config = S3Configuration.fromFields("access", "secret", "endpoint", "region");
+
+ assertNotEquals("not a config", config);
+ }
+
+ public void testEqualsSameInstance() {
+ S3Configuration config = S3Configuration.fromFields("access", "secret", "endpoint", "region");
+
+ assertEquals(config, config);
+ }
+
+ public void testEqualsWithNullFields() {
+ S3Configuration config1 = S3Configuration.fromFields(null, null, "endpoint", null);
+ S3Configuration config2 = S3Configuration.fromFields(null, null, "endpoint", null);
+
+ assertEquals(config1, config2);
+ assertEquals(config1.hashCode(), config2.hashCode());
+ }
+
+ private Literal literal(Object value) {
+ DataType dataType;
+ Object literalValue = value;
+ if (value instanceof String s) {
+ dataType = DataType.KEYWORD;
+ literalValue = new BytesRef(s);
+ } else if (value instanceof Integer) {
+ dataType = DataType.INTEGER;
+ } else if (value instanceof Long) {
+ dataType = DataType.LONG;
+ } else if (value instanceof Double) {
+ dataType = DataType.DOUBLE;
+ } else if (value instanceof Boolean) {
+ dataType = DataType.BOOLEAN;
+ } else {
+ dataType = DataType.KEYWORD;
+ }
+ return new Literal(SOURCE, literalValue, dataType);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/README.md b/x-pack/plugin/esql-datasource-parquet/README.md
new file mode 100644
index 0000000000000..9893430169174
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/README.md
@@ -0,0 +1,122 @@
+# ESQL Parquet Data Source Plugin
+
+This plugin provides Apache Parquet format support for ESQL external data sources.
+
+## Overview
+
+The Parquet plugin enables ESQL to read Parquet files from any storage provider (HTTP, S3, local filesystem). Parquet is a columnar storage format optimized for analytics workloads, providing efficient compression and encoding schemes.
+
+## Features
+
+- **Schema Discovery** - Automatically reads schema from Parquet file metadata
+- **Column Projection** - Only reads requested columns for efficient I/O
+- **Batch Reading** - Configurable batch sizes for memory-efficient processing
+- **Direct Page Conversion** - Converts Parquet data directly to ESQL Page format
+
+## Usage
+
+Once installed, the plugin automatically registers the Parquet format reader. ESQL will use it for any file with a `.parquet` extension:
+
+```sql
+FROM "https://example.com/data/sales.parquet"
+| WHERE region = "EMEA"
+| STATS total = SUM(amount) BY product
+```
+
+```sql
+FROM "s3://my-bucket/warehouse/events.parquet"
+| KEEP timestamp, user_id, event_type
+| SORT timestamp DESC
+| LIMIT 1000
+```
+
+## Dependencies
+
+This plugin bundles the following major dependencies:
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| parquet-hadoop-bundle | 1.16.0 | Parquet file reading and writing |
+| hadoop-client-api | 3.4.1 | Hadoop Configuration class (required by Parquet) |
+| hadoop-client-runtime | 3.4.1 | Hadoop runtime support |
+
+### Why Hadoop Dependencies?
+
+The Hadoop dependencies are required because:
+1. `ParquetFileReader` has method overloads that reference Hadoop `Configuration` in their signatures
+2. `ParquetReadOptions.Builder()` constructor creates `HadoopParquetConfiguration` internally
+3. `parquet-hadoop-bundle` includes shaded Parquet classes but not Hadoop Configuration
+
+## Architecture
+
+```
+┌─────────────────────────────────────────┐
+│ ParquetDataSourcePlugin │
+│ implements DataSourcePlugin │
+└─────────────────┬───────────────────────┘
+ │
+ │ provides
+ ▼
+┌─────────────────────────────────────────┐
+│ ParquetFormatReader │
+│ implements FormatReader │
+│ │
+│ - metadata(StorageObject) │
+│ - read(StorageObject, columns, batch) │
+│ - formatName() → "parquet" │
+│ - fileExtensions() → [".parquet"] │
+└─────────────────┬───────────────────────┘
+ │
+ │ uses
+ ▼
+┌─────────────────────────────────────────┐
+│ ParquetStorageObjectAdapter │
+│ │
+│ Adapts StorageObject to Parquet's │
+│ InputFile interface for random access │
+└─────────────────────────────────────────┘
+```
+
+## Supported Data Types
+
+| Parquet Type | ESQL Type |
+|--------------|-----------|
+| BOOLEAN | BOOLEAN |
+| INT32 | INTEGER |
+| INT64 | LONG |
+| FLOAT | DOUBLE |
+| DOUBLE | DOUBLE |
+| BINARY (UTF8) | KEYWORD |
+| BINARY | KEYWORD (base64) |
+| INT96 (timestamp) | DATETIME |
+| DATE | DATE |
+| TIME | TIME |
+| TIMESTAMP | DATETIME |
+| DECIMAL | DOUBLE |
+| LIST | Not yet supported |
+| MAP | Not yet supported |
+| STRUCT | Not yet supported |
+
+## Building
+
+```bash
+./gradlew :x-pack:plugin:esql-datasource-parquet:build
+```
+
+## Testing
+
+```bash
+# Unit tests
+./gradlew :x-pack:plugin:esql-datasource-parquet:test
+
+# Integration tests
+./gradlew :x-pack:plugin:esql-datasource-parquet:qa:javaRestTest
+```
+
+## Installation
+
+The plugin is bundled with Elasticsearch and enabled by default when the ESQL feature is available.
+
+## License
+
+Elastic License 2.0
diff --git a/x-pack/plugin/esql-datasource-parquet/build.gradle b/x-pack/plugin/esql-datasource-parquet/build.gradle
new file mode 100644
index 0000000000000..6de786766eab1
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/build.gradle
@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.publish'
+
+esplugin {
+ name = 'esql-datasource-parquet'
+ description = 'Parquet format support for ESQL external data sources'
+ classname = 'org.elasticsearch.xpack.esql.datasource.parquet.ParquetDataSourcePlugin'
+ extendedPlugins = ['x-pack-esql']
+}
+
+base {
+ archivesName = 'esql-datasource-parquet'
+}
+
+dependencies {
+ // SPI interfaces from ESQL core
+ compileOnly project(path: xpackModule('esql'))
+ compileOnly project(path: xpackModule('esql-core'))
+ compileOnly project(path: xpackModule('core'))
+ compileOnly project(':server')
+ compileOnly project(xpackModule('esql:compute'))
+
+ // Parquet format support - using parquet-hadoop-bundle to avoid jar hell from duplicate shaded classes
+ implementation('org.apache.parquet:parquet-hadoop-bundle:1.16.0')
+
+ // Hadoop dependencies - required at both compile time and runtime for Parquet operations.
+ //
+ // The Hadoop Configuration class is needed because:
+ // 1. ParquetFileReader has method overloads that reference Configuration in their signatures
+ // 2. ParquetReadOptions.Builder() constructor creates HadoopParquetConfiguration internally,
+ // which requires the Configuration class to be present even when using non-Hadoop code paths
+ // 3. parquet-hadoop-bundle includes shaded Parquet classes but not Hadoop Configuration
+ implementation('org.apache.hadoop:hadoop-client-api:3.4.1')
+ implementation('org.apache.hadoop:hadoop-client-runtime:3.4.1')
+
+ testImplementation project(':test:framework')
+ testImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.named("dependencyLicenses").configure {
+ mapping from: /lucene-.*/, to: 'lucene'
+ mapping from: /parquet-.*/, to: 'parquet'
+ mapping from: /hadoop-.*/, to: 'hadoop'
+}
+
+tasks.named("thirdPartyAudit").configure {
+ ignoreMissingClasses()
+ ignoreViolations(
+ // Hadoop internal uses sun.misc.Unsafe
+ 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm',
+ 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot',
+ 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer',
+ 'org.apache.hadoop.io.FastByteComparisons$LexicographicalComparerHolder$UnsafeComparer$1',
+ 'org.apache.hadoop.io.nativeio.NativeIO',
+ 'org.apache.hadoop.service.launcher.InterruptEscalator',
+ 'org.apache.hadoop.service.launcher.IrqHandler',
+ 'org.apache.hadoop.util.SignalLogger$Handler',
+ // Hadoop shaded Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ // Hadoop shaded Avro uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField',
+ 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
+ // Hadoop shaded Curator Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ 'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor',
+ // Hadoop thirdparty Protobuf uses sun.misc.Unsafe
+ 'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
+ 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
+ // Hadoop thirdparty Guava uses sun.misc.Unsafe
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64',
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$Cell',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
+ 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+ 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+ 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+ 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+ // Parquet shaded hashing uses sun.misc.Unsafe
+ 'shaded.parquet.net.openhft.hashing.HotSpotPrior7u6StringHash',
+ 'shaded.parquet.net.openhft.hashing.LongHashFunction',
+ 'shaded.parquet.net.openhft.hashing.LongTupleHashFunction',
+ 'shaded.parquet.net.openhft.hashing.ModernCompactStringHash',
+ 'shaded.parquet.net.openhft.hashing.ModernHotSpotStringHash',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess$OldUnsafeAccessBigEndian',
+ 'shaded.parquet.net.openhft.hashing.UnsafeAccess$OldUnsafeAccessLittleEndian',
+ 'shaded.parquet.net.openhft.hashing.Util',
+ )
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-LICENSE.txt b/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-NOTICE.txt b/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-NOTICE.txt
new file mode 100644
index 0000000000000..62fc5816c996b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/licenses/hadoop-NOTICE.txt
@@ -0,0 +1,2 @@
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
diff --git a/x-pack/plugin/esql-datasource-parquet/licenses/parquet-LICENSE.txt b/x-pack/plugin/esql-datasource-parquet/licenses/parquet-LICENSE.txt
new file mode 100644
index 0000000000000..f57fe7c0213a9
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/licenses/parquet-LICENSE.txt
@@ -0,0 +1,201 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, ticesnames, and attributions from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Support. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/x-pack/plugin/esql-datasource-parquet/licenses/parquet-NOTICE.txt b/x-pack/plugin/esql-datasource-parquet/licenses/parquet-NOTICE.txt
new file mode 100644
index 0000000000000..63f78a662db1b
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/licenses/parquet-NOTICE.txt
@@ -0,0 +1,13 @@
+Apache Parquet
+Copyright 2014-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project includes code from https://github.com/lemire/JavaFastPFOR
+Copyright 2013 Daniel Lemire and Owen Kaser
+Apache License Version 2.0
+
+This project includes code from https://github.com/lemire/streamvbyte
+Copyright 2017 Daniel Lemire
+Apache License Version 2.0
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/build.gradle b/x-pack/plugin/esql-datasource-parquet/qa/build.gradle
new file mode 100644
index 0000000000000..cb0dac50625c1
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/qa/build.gradle
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
+apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
+
+dependencies {
+ // Test fixtures and spec reader infrastructure
+ javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
+ javaRestTestImplementation project(xpackModule('esql:qa:server'))
+ javaRestTestImplementation project(xpackModule('esql'))
+ javaRestTestImplementation(project(path: xpackModule('esql'), configuration: 'testRuntimeElements'))
+
+ // S3 fixture infrastructure for mocking S3 operations
+ javaRestTestImplementation project(':test:fixtures:s3-fixture')
+ javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
+
+ // S3 datasource provider for discovery tests
+ javaRestTestImplementation project(xpackModule('esql-datasource-s3'))
+
+ // Parquet support - needed for reading test fixtures
+ javaRestTestImplementation('org.apache.parquet:parquet-hadoop-bundle:1.16.0')
+
+ // Repository S3 module for cluster
+ clusterModules project(':modules:repository-s3')
+ clusterPlugins project(':plugins:mapper-size')
+ clusterPlugins project(':plugins:mapper-murmur3')
+
+ // The parquet datasource plugin under test
+ clusterPlugins project(xpackModule('esql-datasource-parquet'))
+ clusterPlugins project(xpackModule('esql-datasource-http'))
+ clusterPlugins project(xpackModule('esql-datasource-s3'))
+}
+
+// The parquet fixtures (employees.parquet and parquet-basic.csv-spec) are included
+// directly in this module's javaRestTest/resources directory
+
+// S3GlobDiscoveryIT extends ESTestCase (not ESRestTestCase) since it tests S3StorageProvider
+// directly against the S3HttpFixture without needing an Elasticsearch cluster.
+tasks.named('javaRestTestTestingConventions').configure {
+ baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+ baseClass 'org.elasticsearch.test.ESTestCase'
+}
+
+tasks.named("forbiddenPatterns").configure {
+ exclude '**/*.parquet'
+}
+
+tasks.named('javaRestTest') {
+ usesDefaultDistribution("to be triaged")
+ maxParallelForks = 1
+
+ // Increase timeouts for S3/Parquet operations which may take longer than standard queries
+ systemProperty 'tests.rest.client_timeout', '60'
+ systemProperty 'tests.rest.socket_timeout', '60'
+
+ // Enable more verbose logging for debugging
+ testLogging {
+ events = ["passed", "skipped", "failed"]
+ exceptionFormat = "full"
+ showStandardStreams = false
+ }
+}
+
+restResources {
+ restApi {
+ include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'cluster', 'capabilities', 'index'
+ }
+ restTests {
+ includeXpack 'esql'
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/Clusters.java b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/Clusters.java
new file mode 100644
index 0000000000000..70a5242b221a8
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/Clusters.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.parquet;
+
+import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.ACCESS_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.SECRET_KEY;
+
+/**
+ * Cluster configuration for Parquet integration tests.
+ */
+public class Clusters {
+
+ public static ElasticsearchCluster testCluster(Supplier s3EndpointSupplier, LocalClusterConfigProvider configProvider) {
+ return ElasticsearchCluster.local()
+ .distribution(DistributionType.DEFAULT)
+ .shared(true)
+ // Enable S3 repository plugin for S3 access
+ .module("repository-s3")
+ // Basic cluster settings
+ .setting("xpack.security.enabled", "false")
+ .setting("xpack.license.self_generated.type", "trial")
+ // Disable ML to avoid native code loading issues in some environments
+ .setting("xpack.ml.enabled", "false")
+ // Allow the LOCAL storage backend to read fixture files from the test resources directory.
+ // The esql-datasource-http plugin's entitlement policy uses shared_repo for file read access.
+ .setting("path.repo", fixturesPath())
+ // S3 client configuration for accessing the S3HttpFixture
+ .setting("s3.client.default.endpoint", s3EndpointSupplier)
+ // S3 credentials must be stored in keystore, not as regular settings
+ .keystore("s3.client.default.access_key", ACCESS_KEY)
+ .keystore("s3.client.default.secret_key", SECRET_KEY)
+ // Disable SSL for HTTP fixture
+ .setting("s3.client.default.protocol", "http")
+ // Disable AWS SDK profile file loading by pointing to non-existent files
+ // This prevents the SDK from trying to read ~/.aws/credentials and ~/.aws/config
+ // which would violate Elasticsearch entitlements
+ .environment("AWS_CONFIG_FILE", "/dev/null/aws/config")
+ .environment("AWS_SHARED_CREDENTIALS_FILE", "/dev/null/aws/credentials")
+ // Arrow's unsafe memory allocator requires access to java.nio internals
+ .jvmArg("--add-opens=java.base/java.nio=ALL-UNNAMED")
+ // Configure Arrow to use unsafe memory allocator instead of netty
+ // This must be set as a JVM arg to take effect before any Arrow classes are loaded
+ .jvmArg("-Darrow.allocation.manager.type=Unsafe")
+ // Apply any additional configuration
+ .apply(() -> configProvider)
+ .build();
+ }
+
+ public static ElasticsearchCluster testCluster(Supplier s3EndpointSupplier) {
+ return testCluster(s3EndpointSupplier, config -> {});
+ }
+
+ private static String fixturesPath() {
+ URL resourceUrl = Clusters.class.getResource("/iceberg-fixtures");
+ if (resourceUrl != null && resourceUrl.getProtocol().equals("file")) {
+ try {
+ return PathUtils.get(resourceUrl.toURI()).toAbsolutePath().toString();
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Failed to resolve fixtures path", e);
+ }
+ }
+ // Fall back to a safe default; LOCAL tests will fail gracefully
+ return "/tmp";
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/ParquetFormatSpecIT.java b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/ParquetFormatSpecIT.java
new file mode 100644
index 0000000000000..71a9d3c7b32e5
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/ParquetFormatSpecIT.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.parquet;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.qa.rest.AbstractExternalSourceSpecTestCase;
+import org.junit.ClassRule;
+
+import java.util.List;
+
+/**
+ * Parameterized integration tests for standalone Parquet files.
+ * Each csv-spec test is run against every configured storage backend (S3, HTTP, LOCAL).
+ */
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class ParquetFormatSpecIT extends AbstractExternalSourceSpecTestCase {
+
+ @ClassRule
+ public static ElasticsearchCluster cluster = Clusters.testCluster(() -> s3Fixture.getAddress());
+
+ public ParquetFormatSpecIT(
+ String fileName,
+ String groupName,
+ String testName,
+ Integer lineNumber,
+ CsvTestCase testCase,
+ String instructions,
+ StorageBackend storageBackend
+ ) {
+ super(fileName, groupName, testName, lineNumber, testCase, instructions, storageBackend, "parquet");
+ }
+
+ @Override
+ protected String getTestRestCluster() {
+ return cluster.getHttpAddresses();
+ }
+
+ @ParametersFactory(argumentFormatting = "csv-spec:%2$s.%3$s [%7$s]")
+ public static List readScriptSpec() throws Exception {
+ return readExternalSpecTests("/external-*.csv-spec");
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/S3GlobDiscoveryIT.java b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/S3GlobDiscoveryIT.java
new file mode 100644
index 0000000000000..29d526ed8ea44
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/parquet/S3GlobDiscoveryIT.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.parquet;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.datasource.s3.S3Configuration;
+import org.elasticsearch.xpack.esql.datasource.s3.S3StorageProvider;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.DataSourcesS3HttpFixture;
+import org.elasticsearch.xpack.esql.datasources.StorageEntry;
+import org.elasticsearch.xpack.esql.datasources.StorageIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.ACCESS_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.BUCKET;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.SECRET_KEY;
+
+/**
+ * S3 discovery tests using S3HttpFixture with empty blobs.
+ * Validates that S3StorageProvider.listObjects() returns correct entries
+ * and that glob-style filtering works against S3 listings.
+ */
+public class S3GlobDiscoveryIT extends ESTestCase {
+
+ @ClassRule
+ public static DataSourcesS3HttpFixture s3Fixture = new DataSourcesS3HttpFixture();
+
+ private static S3StorageProvider provider;
+
+ private static final String DISCOVER_PREFIX = "warehouse/discover";
+
+ @BeforeClass
+ public static void setupProvider() {
+ // Upload empty blobs for discovery
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), DISCOVER_PREFIX + "/flat/a.parquet", new byte[0]);
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), DISCOVER_PREFIX + "/flat/b.parquet", new byte[0]);
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), DISCOVER_PREFIX + "/flat/c.csv", new byte[0]);
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), DISCOVER_PREFIX + "/nested/x/d.parquet", new byte[0]);
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), DISCOVER_PREFIX + "/nested/y/e.parquet", new byte[0]);
+
+ S3Configuration config = S3Configuration.fromFields(ACCESS_KEY, SECRET_KEY, s3Fixture.getAddress(), "us-east-1");
+ provider = new S3StorageProvider(config);
+ }
+
+ @AfterClass
+ public static void cleanupProvider() throws Exception {
+ if (provider != null) {
+ provider.close();
+ provider = null;
+ }
+ }
+
+ public void testS3FlatListing() throws IOException {
+ StoragePath prefix = StoragePath.of("s3://" + BUCKET + "/" + DISCOVER_PREFIX + "/flat");
+ List entries = collectAll(provider.listObjects(prefix, false));
+
+ List names = entries.stream().map(e -> e.path().objectName()).sorted().toList();
+ assertEquals(List.of("a.parquet", "b.parquet", "c.csv"), names);
+ }
+
+ public void testS3FlatGlobFiltering() throws IOException {
+ StoragePath prefix = StoragePath.of("s3://" + BUCKET + "/" + DISCOVER_PREFIX + "/flat");
+ List entries = collectAll(provider.listObjects(prefix, false));
+
+ // Simulate *.parquet glob filtering
+ Pattern parquetPattern = Pattern.compile("[^/]*\\.parquet");
+ List matched = new ArrayList<>();
+ for (StorageEntry e : entries) {
+ if (parquetPattern.matcher(e.path().objectName()).matches()) {
+ matched.add(e);
+ }
+ }
+
+ assertEquals(2, matched.size());
+ }
+
+ public void testS3RecursiveGlobFiltering() throws IOException {
+ // S3 is flat — listing with a prefix returns all objects under it
+ StoragePath prefix = StoragePath.of("s3://" + BUCKET + "/" + DISCOVER_PREFIX);
+ List entries = collectAll(provider.listObjects(prefix, true));
+
+ // Simulate **/*.parquet: match any .parquet file at any depth
+ String prefixStr = "s3://" + BUCKET + "/" + DISCOVER_PREFIX + "/";
+ List matched = new ArrayList<>();
+ for (StorageEntry e : entries) {
+ String fullPath = e.path().toString();
+ String relativePath = fullPath.startsWith(prefixStr) ? fullPath.substring(prefixStr.length()) : e.path().objectName();
+ if (relativePath.endsWith(".parquet")) {
+ matched.add(e);
+ }
+ }
+
+ assertEquals(4, matched.size());
+ }
+
+ public void testS3NoMatchReturnsEmpty() throws IOException {
+ StoragePath prefix = StoragePath.of("s3://" + BUCKET + "/" + DISCOVER_PREFIX + "/flat");
+ List entries = collectAll(provider.listObjects(prefix, false));
+
+ // Simulate *.json glob filtering — no matches expected
+ Pattern jsonPattern = Pattern.compile("[^/]*\\.json");
+ List matched = new ArrayList<>();
+ for (StorageEntry e : entries) {
+ if (jsonPattern.matcher(e.path().objectName()).matches()) {
+ matched.add(e);
+ }
+ }
+
+ assertEquals(0, matched.size());
+ }
+
+ public void testS3BraceAlternativesFiltering() throws IOException {
+ StoragePath prefix = StoragePath.of("s3://" + BUCKET + "/" + DISCOVER_PREFIX + "/flat");
+ List entries = collectAll(provider.listObjects(prefix, false));
+
+ // Simulate *.{parquet,csv} glob filtering
+ Pattern bracePattern = Pattern.compile("[^/]*\\.(?:parquet|csv)");
+ List matched = new ArrayList<>();
+ for (StorageEntry e : entries) {
+ if (bracePattern.matcher(e.path().objectName()).matches()) {
+ matched.add(e);
+ }
+ }
+
+ assertEquals(3, matched.size());
+ }
+
+ private static List collectAll(StorageIterator iterator) throws IOException {
+ List entries = new ArrayList<>();
+ try (iterator) {
+ while (iterator.hasNext()) {
+ entries.add(iterator.next());
+ }
+ }
+ return entries;
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_01.parquet b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_01.parquet
new file mode 100644
index 0000000000000..e1073b577b15e
Binary files /dev/null and b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_01.parquet differ
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_02.parquet b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_02.parquet
new file mode 100644
index 0000000000000..33ea9ab32d167
Binary files /dev/null and b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/multifile/employees_02.parquet differ
diff --git a/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet
new file mode 100644
index 0000000000000..40c723aa7d812
Binary files /dev/null and b/x-pack/plugin/esql-datasource-parquet/qa/src/javaRestTest/resources/iceberg-fixtures/standalone/employees.parquet differ
diff --git a/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetDataSourcePlugin.java b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetDataSourcePlugin.java
new file mode 100644
index 0000000000000..c65cb34657495
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetDataSourcePlugin.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.parquet;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReaderFactory;
+
+import java.util.Map;
+
+/**
+ * Data source plugin that provides Parquet format support for ESQL external data sources.
+ *
+ * This plugin provides:
+ *
+ * Parquet format reader for reading Parquet files from any storage provider
+ *
+ *
+ * The Parquet format reader uses Apache Parquet's native ParquetFileReader with
+ * Iceberg's schema conversion utilities. It supports:
+ *
+ * Schema discovery from Parquet file metadata
+ * Column projection for efficient reads
+ * Batch reading with configurable batch sizes
+ * Direct conversion to ESQL Page format
+ *
+ *
+ * Heavy dependencies (Parquet, Hadoop, Iceberg, Arrow) are isolated in this module
+ * to avoid jar hell issues in the core ESQL plugin.
+ */
+public class ParquetDataSourcePlugin extends Plugin implements DataSourcePlugin {
+
+ @Override
+ public Map formatReaders(Settings settings) {
+ return Map.of("parquet", (s, blockFactory) -> new ParquetFormatReader(blockFactory));
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReader.java b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReader.java
new file mode 100644
index 0000000000000..0fbcfa2df03be
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReader.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.parquet;
+
+import org.apache.parquet.ParquetReadOptions;
+import org.apache.parquet.column.page.PageReadStore;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.io.ColumnIOFactory;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.RecordReader;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReader;
+import org.elasticsearch.xpack.esql.datasources.spi.SimpleSourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.SourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+/**
+ * FormatReader implementation for Parquet files.
+ *
+ * Uses Parquet's native ParquetFileReader with our StorageObject abstraction.
+ * Produces ESQL Page batches directly without requiring Arrow as an intermediate format.
+ *
+ *
Key features:
+ *
+ * Works with any StorageProvider (HTTP, S3, local)
+ * Efficient columnar reading with column projection
+ * No Hadoop dependencies in the core path
+ * Direct conversion from Parquet to ESQL blocks
+ *
+ */
+public class ParquetFormatReader implements FormatReader {
+
+ private final BlockFactory blockFactory;
+
+ public ParquetFormatReader(BlockFactory blockFactory) {
+ this.blockFactory = blockFactory;
+ }
+
+ @Override
+ public SourceMetadata metadata(StorageObject object) throws IOException {
+ List schema = readSchema(object);
+ return new SimpleSourceMetadata(schema, formatName(), object.path().toString());
+ }
+
+ private List readSchema(StorageObject object) throws IOException {
+ // Adapt StorageObject to Parquet InputFile
+ org.apache.parquet.io.InputFile parquetInputFile = new ParquetStorageObjectAdapter(object);
+
+ // Build ParquetReadOptions with SKIP_ROW_GROUPS to only read schema metadata
+ ParquetReadOptions options = ParquetReadOptions.builder().withMetadataFilter(ParquetMetadataConverter.SKIP_ROW_GROUPS).build();
+
+ try (ParquetFileReader reader = ParquetFileReader.open(parquetInputFile, options)) {
+ org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = reader.getFileMetaData();
+ MessageType parquetSchema = fileMetaData.getSchema();
+
+ // Convert Parquet schema directly to ESQL Attributes
+ return convertParquetSchemaToAttributes(parquetSchema);
+ }
+ }
+
+ @Override
+ public CloseableIterator read(StorageObject object, List projectedColumns, int batchSize) throws IOException {
+ // Adapt StorageObject to Parquet InputFile
+ org.apache.parquet.io.InputFile parquetInputFile = new ParquetStorageObjectAdapter(object);
+
+ // Build ParquetReadOptions for data reading
+ ParquetReadOptions options = ParquetReadOptions.builder().build();
+
+ // Open the Parquet file reader
+ ParquetFileReader reader = ParquetFileReader.open(parquetInputFile, options);
+
+ // Get the schema
+ org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = reader.getFileMetaData();
+ MessageType parquetSchema = fileMetaData.getSchema();
+ List attributes = convertParquetSchemaToAttributes(parquetSchema);
+
+ // Filter attributes based on projection
+ List projectedAttributes;
+ if (projectedColumns == null || projectedColumns.isEmpty()) {
+ projectedAttributes = attributes;
+ } else {
+ projectedAttributes = new ArrayList<>();
+ Map attributeMap = new HashMap<>();
+ for (Attribute attr : attributes) {
+ attributeMap.put(attr.name(), attr);
+ }
+ for (String columnName : projectedColumns) {
+ Attribute attr = attributeMap.get(columnName);
+ if (attr != null) {
+ projectedAttributes.add(attr);
+ }
+ }
+ }
+
+ return new ParquetPageIterator(reader, parquetSchema, projectedAttributes, batchSize, blockFactory);
+ }
+
+ @Override
+ public String formatName() {
+ return "parquet";
+ }
+
+ @Override
+ public List fileExtensions() {
+ return List.of(".parquet", ".parq");
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to close at the reader level
+ }
+
+ private List convertParquetSchemaToAttributes(MessageType schema) {
+ List attributes = new ArrayList<>();
+ for (Type field : schema.getFields()) {
+ String name = field.getName();
+ DataType esqlType = convertParquetTypeToEsql(field);
+ attributes.add(new ReferenceAttribute(Source.EMPTY, name, esqlType));
+ }
+ return attributes;
+ }
+
+ private DataType convertParquetTypeToEsql(Type parquetType) {
+ if (parquetType.isPrimitive() == false) {
+ return DataType.UNSUPPORTED; // Complex types not yet supported
+ }
+ PrimitiveType primitive = parquetType.asPrimitiveType();
+ LogicalTypeAnnotation logical = primitive.getLogicalTypeAnnotation();
+
+ return switch (primitive.getPrimitiveTypeName()) {
+ case BOOLEAN -> DataType.BOOLEAN;
+ case INT32 -> logical instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation ? DataType.DATETIME : DataType.INTEGER;
+ case INT64 -> logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation ? DataType.DATETIME : DataType.LONG;
+ case FLOAT, DOUBLE -> DataType.DOUBLE;
+ case BINARY, FIXED_LEN_BYTE_ARRAY -> {
+ // Check for STRING logical type
+ if (logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) {
+ yield DataType.KEYWORD;
+ }
+ // Default binary to keyword
+ yield DataType.KEYWORD;
+ }
+ default -> DataType.UNSUPPORTED;
+ };
+ }
+
+ private static class ParquetPageIterator implements CloseableIterator {
+ private final ParquetFileReader reader;
+ private final MessageType parquetSchema;
+ private final List attributes;
+ private final int batchSize;
+ private final MessageColumnIO columnIO;
+ private final BlockFactory blockFactory;
+
+ private PageReadStore currentRowGroup;
+ private RecordReader recordReader;
+ private long rowsRemainingInGroup;
+ private boolean exhausted = false;
+
+ ParquetPageIterator(
+ ParquetFileReader reader,
+ MessageType parquetSchema,
+ List attributes,
+ int batchSize,
+ BlockFactory blockFactory
+ ) {
+ this.reader = reader;
+ this.parquetSchema = parquetSchema;
+ this.attributes = attributes;
+ this.batchSize = batchSize;
+ this.columnIO = new ColumnIOFactory().getColumnIO(parquetSchema);
+ this.blockFactory = blockFactory;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (exhausted) {
+ return false;
+ }
+ // Check if we have rows in current group or can read more groups
+ if (rowsRemainingInGroup > 0) {
+ return true;
+ }
+ // Try to read next row group
+ try {
+ currentRowGroup = reader.readNextRowGroup();
+ if (currentRowGroup == null) {
+ exhausted = true;
+ return false;
+ }
+ rowsRemainingInGroup = currentRowGroup.getRowCount();
+ recordReader = columnIO.getRecordReader(currentRowGroup, new GroupRecordConverter(parquetSchema));
+ return rowsRemainingInGroup > 0;
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read Parquet row group", e);
+ }
+ }
+
+ @Override
+ public Page next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+
+ try {
+ // Read records up to batch size
+ List batch = new ArrayList<>(batchSize);
+ int rowsToRead = (int) Math.min(batchSize, rowsRemainingInGroup);
+
+ for (int i = 0; i < rowsToRead; i++) {
+ Group group = recordReader.read();
+ if (group != null) {
+ batch.add(group);
+ rowsRemainingInGroup--;
+ }
+ }
+
+ if (batch.isEmpty()) {
+ throw new NoSuchElementException("No more records");
+ }
+
+ // Convert batch to ESQL Page
+ return convertToPage(batch);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create Page batch", e);
+ }
+ }
+
+ private Page convertToPage(List batch) {
+ int rowCount = batch.size();
+ Block[] blocks = new Block[attributes.size()];
+
+ // Create a block for each attribute
+ for (int col = 0; col < attributes.size(); col++) {
+ Attribute attribute = attributes.get(col);
+ String fieldName = attribute.name();
+ DataType dataType = attribute.dataType();
+
+ blocks[col] = createBlock(batch, fieldName, dataType, rowCount);
+ }
+
+ return new Page(blocks);
+ }
+
+ private Block createBlock(List batch, String fieldName, DataType dataType, int rowCount) {
+ // Find field index in Parquet schema
+ int fieldIndex = findFieldIndex(batch.get(0), fieldName);
+ if (fieldIndex == -1) {
+ // Field not found, return null block
+ return blockFactory.newConstantNullBlock(rowCount);
+ }
+
+ return switch (dataType) {
+ case BOOLEAN -> createBooleanBlock(batch, fieldName, fieldIndex, rowCount);
+ case INTEGER -> createIntBlock(batch, fieldName, fieldIndex, rowCount);
+ case LONG -> createLongBlock(batch, fieldName, fieldIndex, rowCount);
+ case DOUBLE -> createDoubleBlock(batch, fieldName, fieldIndex, rowCount);
+ case KEYWORD, TEXT -> createBytesRefBlock(batch, fieldName, fieldIndex, rowCount);
+ case DATETIME -> createLongBlock(batch, fieldName, fieldIndex, rowCount); // Timestamps as longs
+ default -> blockFactory.newConstantNullBlock(rowCount);
+ };
+ }
+
+ private int findFieldIndex(Group group, String fieldName) {
+ org.apache.parquet.schema.GroupType groupType = group.getType();
+ int fieldCount = groupType.getFieldCount();
+ for (int i = 0; i < fieldCount; i++) {
+ Type fieldType = groupType.getType(i);
+ String name = fieldType.getName();
+ if (name.equals(fieldName)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private Block createBooleanBlock(List batch, String fieldName, int fieldIndex, int rowCount) {
+ try (var builder = blockFactory.newBooleanBlockBuilder(rowCount)) {
+ for (Group group : batch) {
+ if (group.getFieldRepetitionCount(fieldIndex) == 0) {
+ builder.appendNull();
+ } else {
+ builder.appendBoolean(group.getBoolean(fieldName, 0));
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ private Block createIntBlock(List batch, String fieldName, int fieldIndex, int rowCount) {
+ try (var builder = blockFactory.newIntBlockBuilder(rowCount)) {
+ for (Group group : batch) {
+ if (group.getFieldRepetitionCount(fieldIndex) == 0) {
+ builder.appendNull();
+ } else {
+ builder.appendInt(group.getInteger(fieldName, 0));
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ private Block createLongBlock(List batch, String fieldName, int fieldIndex, int rowCount) {
+ try (var builder = blockFactory.newLongBlockBuilder(rowCount)) {
+ for (Group group : batch) {
+ if (group.getFieldRepetitionCount(fieldIndex) == 0) {
+ builder.appendNull();
+ } else {
+ builder.appendLong(group.getLong(fieldName, 0));
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ private Block createDoubleBlock(List batch, String fieldName, int fieldIndex, int rowCount) {
+ try (var builder = blockFactory.newDoubleBlockBuilder(rowCount)) {
+ for (Group group : batch) {
+ if (group.getFieldRepetitionCount(fieldIndex) == 0) {
+ builder.appendNull();
+ } else {
+ // Handle both float and double
+ org.apache.parquet.schema.GroupType groupType = group.getType();
+ org.apache.parquet.schema.Type fieldType = groupType.getType(fieldIndex);
+ PrimitiveType primitiveType = fieldType.asPrimitiveType();
+ PrimitiveType.PrimitiveTypeName typeName = primitiveType.getPrimitiveTypeName();
+ if (typeName == PrimitiveType.PrimitiveTypeName.FLOAT) {
+ builder.appendDouble(group.getFloat(fieldName, 0));
+ } else {
+ builder.appendDouble(group.getDouble(fieldName, 0));
+ }
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ private Block createBytesRefBlock(List batch, String fieldName, int fieldIndex, int rowCount) {
+ try (var builder = blockFactory.newBytesRefBlockBuilder(rowCount)) {
+ for (Group group : batch) {
+ if (group.getFieldRepetitionCount(fieldIndex) == 0) {
+ builder.appendNull();
+ } else {
+ String value = group.getString(fieldName, 0);
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ builder.appendBytesRef(new org.apache.lucene.util.BytesRef(bytes));
+ }
+ }
+ return builder.build();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapter.java b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapter.java
new file mode 100644
index 0000000000000..a8f3ee3ca92e3
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/main/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapter.java
@@ -0,0 +1,215 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.parquet;
+
+import org.apache.parquet.io.SeekableInputStream;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Adapter that wraps a StorageObject to implement Parquet's InputFile interface.
+ * This allows using our storage abstraction with Parquet's ParquetFileReader.
+ *
+ * Key features:
+ *
+ * Converts StorageObject's range-based reads to Parquet's seekable stream interface
+ * Supports efficient random access for columnar format reading
+ * No Hadoop dependencies - uses pure Java InputStream
+ *
+ */
+public class ParquetStorageObjectAdapter implements org.apache.parquet.io.InputFile {
+ private final StorageObject storageObject;
+
+ /**
+ * Creates an adapter for the given StorageObject.
+ *
+ * @param storageObject the storage object to adapt
+ */
+ public ParquetStorageObjectAdapter(StorageObject storageObject) {
+ if (storageObject == null) {
+ throw new IllegalArgumentException("storageObject cannot be null");
+ }
+ this.storageObject = storageObject;
+ }
+
+ @Override
+ public long getLength() throws IOException {
+ return storageObject.length();
+ }
+
+ @Override
+ public SeekableInputStream newStream() throws IOException {
+ return new StorageObjectSeekableInputStream(storageObject);
+ }
+
+ /**
+ * SeekableInputStream implementation that uses StorageObject's range-based reads.
+ *
+ * This implementation provides efficient random access by:
+ *
+ * Tracking current position in the stream
+ * Using range reads for seek operations
+ * Buffering data from the current stream until a seek is needed
+ *
+ */
+ private static class StorageObjectSeekableInputStream extends SeekableInputStream {
+ private final StorageObject storageObject;
+ private InputStream currentStream;
+ private long position;
+ private long streamStartPosition;
+ private final long length;
+
+ StorageObjectSeekableInputStream(StorageObject storageObject) throws IOException {
+ this.storageObject = storageObject;
+ this.length = storageObject.length();
+ this.position = 0;
+ this.streamStartPosition = 0;
+ // Open initial stream from beginning
+ this.currentStream = storageObject.newStream();
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return position;
+ }
+
+ @Override
+ public void seek(long newPos) throws IOException {
+ if (newPos < 0) {
+ throw new IOException("Cannot seek to negative position: " + newPos);
+ }
+ if (newPos > length) {
+ throw new IOException("Cannot seek beyond end of file: " + newPos + " > " + length);
+ }
+
+ // If we're seeking within the current stream, try to skip forward
+ if (newPos >= streamStartPosition && newPos >= position) {
+ long skipAmount = newPos - position;
+ if (skipAmount > 0) {
+ long skipped = currentStream.skip(skipAmount);
+ if (skipped != skipAmount) {
+ // Skip failed, need to reopen stream
+ reopenStreamAt(newPos);
+ } else {
+ position = newPos;
+ }
+ }
+ // If newPos == position, we're already there
+ return;
+ }
+
+ // For backward seeks or large forward seeks, reopen the stream
+ reopenStreamAt(newPos);
+ }
+
+ /**
+ * Reopens the stream at the specified position using a range read.
+ */
+ private void reopenStreamAt(long newPos) throws IOException {
+ // Close current stream
+ if (currentStream != null) {
+ currentStream.close();
+ }
+
+ // Open new stream from the target position to the end
+ long remainingBytes = length - newPos;
+ currentStream = storageObject.newStream(newPos, remainingBytes);
+ streamStartPosition = newPos;
+ position = newPos;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int b = currentStream.read();
+ if (b >= 0) {
+ position++;
+ }
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int bytesRead = currentStream.read(b, off, len);
+ if (bytesRead > 0) {
+ position += bytesRead;
+ }
+ return bytesRead;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ long skipped = currentStream.skip(n);
+ position += skipped;
+ return skipped;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return currentStream.available();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (currentStream != null) {
+ currentStream.close();
+ currentStream = null;
+ }
+ }
+
+ @Override
+ public void readFully(byte[] bytes) throws IOException {
+ readFully(bytes, 0, bytes.length);
+ }
+
+ @Override
+ public void readFully(byte[] bytes, int start, int len) throws IOException {
+ int offset = start;
+ int remaining = len;
+ while (remaining > 0) {
+ int bytesRead = read(bytes, offset, remaining);
+ if (bytesRead < 0) {
+ throw new IOException("Reached end of stream before reading " + len + " bytes");
+ }
+ offset += bytesRead;
+ remaining -= bytesRead;
+ }
+ }
+
+ @Override
+ public int read(java.nio.ByteBuffer buf) throws IOException {
+ if (buf.hasRemaining() == false) {
+ return 0;
+ }
+
+ int bytesToRead = buf.remaining();
+ byte[] temp = new byte[bytesToRead];
+ int bytesRead = read(temp, 0, bytesToRead);
+
+ if (bytesRead > 0) {
+ buf.put(temp, 0, bytesRead);
+ }
+
+ return bytesRead;
+ }
+
+ @Override
+ public void readFully(java.nio.ByteBuffer buf) throws IOException {
+ int remaining = buf.remaining();
+ byte[] temp = new byte[remaining];
+ readFully(temp, 0, remaining);
+ buf.put(temp);
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin b/x-pack/plugin/esql-datasource-parquet/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
new file mode 100644
index 0000000000000..1bcccdf0b5090
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
@@ -0,0 +1 @@
+org.elasticsearch.xpack.esql.datasource.parquet.ParquetDataSourcePlugin
diff --git a/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReaderTests.java b/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReaderTests.java
new file mode 100644
index 0000000000000..127e15b457ed0
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetFormatReaderTests.java
@@ -0,0 +1,473 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.parquet;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.PositionOutputStream;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.SourceMetadata;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.Instant;
+import java.util.List;
+
+public class ParquetFormatReaderTests extends ESTestCase {
+
+ private BlockFactory blockFactory;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ blockFactory = BlockFactory.getInstance(new NoopCircuitBreaker("test-noop"), BigArrays.NON_RECYCLING_INSTANCE);
+ }
+
+ public void testFormatName() {
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+ assertEquals("parquet", reader.formatName());
+ }
+
+ public void testFileExtensions() {
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+ List extensions = reader.fileExtensions();
+ assertEquals(2, extensions.size());
+ assertTrue(extensions.contains(".parquet"));
+ assertTrue(extensions.contains(".parq"));
+ }
+
+ public void testReadSchemaFromSimpleParquet() throws Exception {
+ // Create a simple parquet file with known schema
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("id")
+ .required(PrimitiveType.PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("name")
+ .required(PrimitiveType.PrimitiveTypeName.INT32)
+ .named("age")
+ .required(PrimitiveType.PrimitiveTypeName.BOOLEAN)
+ .named("active")
+ .named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("id", 1L);
+ group1.add("name", "Alice");
+ group1.add("age", 30);
+ group1.add("active", true);
+ return List.of(group1);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ SourceMetadata metadata = reader.metadata(storageObject);
+ List attributes = metadata.schema();
+
+ assertEquals(4, attributes.size());
+
+ assertEquals("id", attributes.get(0).name());
+ assertEquals(DataType.LONG, attributes.get(0).dataType());
+
+ assertEquals("name", attributes.get(1).name());
+ assertEquals(DataType.KEYWORD, attributes.get(1).dataType());
+
+ assertEquals("age", attributes.get(2).name());
+ assertEquals(DataType.INTEGER, attributes.get(2).dataType());
+
+ assertEquals("active", attributes.get(3).name());
+ assertEquals(DataType.BOOLEAN, attributes.get(3).dataType());
+ }
+
+ public void testReadDataFromSimpleParquet() throws Exception {
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("id")
+ .required(PrimitiveType.PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("name")
+ .required(PrimitiveType.PrimitiveTypeName.DOUBLE)
+ .named("score")
+ .named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("id", 1L);
+ group1.add("name", "Alice");
+ group1.add("score", 95.5);
+
+ Group group2 = factory.newGroup();
+ group2.add("id", 2L);
+ group2.add("name", "Bob");
+ group2.add("score", 87.3);
+
+ Group group3 = factory.newGroup();
+ group3.add("id", 3L);
+ group3.add("name", "Charlie");
+ group3.add("score", 92.1);
+
+ return List.of(group1, group2, group3);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(storageObject, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(3, page.getPositionCount());
+ assertEquals(3, page.getBlockCount());
+
+ // Check first row
+ assertEquals(1L, ((LongBlock) page.getBlock(0)).getLong(0));
+ assertEquals(new BytesRef("Alice"), ((BytesRefBlock) page.getBlock(1)).getBytesRef(0, new BytesRef()));
+ assertEquals(95.5, ((DoubleBlock) page.getBlock(2)).getDouble(0), 0.001);
+
+ // Check second row
+ assertEquals(2L, ((LongBlock) page.getBlock(0)).getLong(1));
+ assertEquals(new BytesRef("Bob"), ((BytesRefBlock) page.getBlock(1)).getBytesRef(1, new BytesRef()));
+ assertEquals(87.3, ((DoubleBlock) page.getBlock(2)).getDouble(1), 0.001);
+
+ // Check third row
+ assertEquals(3L, ((LongBlock) page.getBlock(0)).getLong(2));
+ assertEquals(new BytesRef("Charlie"), ((BytesRefBlock) page.getBlock(1)).getBytesRef(2, new BytesRef()));
+ assertEquals(92.1, ((DoubleBlock) page.getBlock(2)).getDouble(2), 0.001);
+
+ assertFalse(iterator.hasNext());
+ }
+ }
+
+ public void testReadWithColumnProjection() throws Exception {
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("id")
+ .required(PrimitiveType.PrimitiveTypeName.BINARY)
+ .as(LogicalTypeAnnotation.stringType())
+ .named("name")
+ .required(PrimitiveType.PrimitiveTypeName.DOUBLE)
+ .named("score")
+ .named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("id", 1L);
+ group1.add("name", "Alice");
+ group1.add("score", 95.5);
+
+ Group group2 = factory.newGroup();
+ group2.add("id", 2L);
+ group2.add("name", "Bob");
+ group2.add("score", 87.3);
+
+ return List.of(group1, group2);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ // Project only name and score columns
+ try (CloseableIterator iterator = reader.read(storageObject, List.of("name", "score"), 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(2, page.getPositionCount());
+ assertEquals(2, page.getBlockCount()); // Only 2 projected columns
+
+ // Check values - note: order matches projection order
+ assertEquals(new BytesRef("Alice"), ((BytesRefBlock) page.getBlock(0)).getBytesRef(0, new BytesRef()));
+ assertEquals(95.5, ((DoubleBlock) page.getBlock(1)).getDouble(0), 0.001);
+
+ assertEquals(new BytesRef("Bob"), ((BytesRefBlock) page.getBlock(0)).getBytesRef(1, new BytesRef()));
+ assertEquals(87.3, ((DoubleBlock) page.getBlock(1)).getDouble(1), 0.001);
+ }
+ }
+
+ public void testReadWithBatching() throws Exception {
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("id")
+ .required(PrimitiveType.PrimitiveTypeName.INT32)
+ .named("value")
+ .named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ List groups = new java.util.ArrayList<>();
+ for (int i = 1; i <= 25; i++) {
+ Group group = factory.newGroup();
+ group.add("id", (long) i);
+ group.add("value", i * 10);
+ groups.add(group);
+ }
+ return groups;
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ int batchSize = 10;
+ int totalRows = 0;
+
+ try (CloseableIterator iterator = reader.read(storageObject, null, batchSize)) {
+ while (iterator.hasNext()) {
+ Page page = iterator.next();
+ totalRows += page.getPositionCount();
+ }
+ }
+
+ assertEquals(25, totalRows);
+ }
+
+ public void testReadBooleanColumn() throws Exception {
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("id")
+ .required(PrimitiveType.PrimitiveTypeName.BOOLEAN)
+ .named("active")
+ .named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("id", 1L);
+ group1.add("active", true);
+
+ Group group2 = factory.newGroup();
+ group2.add("id", 2L);
+ group2.add("active", false);
+
+ return List.of(group1, group2);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(storageObject, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(2, page.getPositionCount());
+
+ assertTrue(((BooleanBlock) page.getBlock(1)).getBoolean(0));
+ assertFalse(((BooleanBlock) page.getBlock(1)).getBoolean(1));
+ }
+ }
+
+ public void testReadIntegerColumn() throws Exception {
+ MessageType schema = Types.buildMessage().required(PrimitiveType.PrimitiveTypeName.INT32).named("count").named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("count", 100);
+
+ Group group2 = factory.newGroup();
+ group2.add("count", 200);
+
+ Group group3 = factory.newGroup();
+ group3.add("count", 300);
+
+ return List.of(group1, group2, group3);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(storageObject, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(3, page.getPositionCount());
+
+ assertEquals(100, ((IntBlock) page.getBlock(0)).getInt(0));
+ assertEquals(200, ((IntBlock) page.getBlock(0)).getInt(1));
+ assertEquals(300, ((IntBlock) page.getBlock(0)).getInt(2));
+ }
+ }
+
+ public void testReadFloatColumn() throws Exception {
+ MessageType schema = Types.buildMessage().required(PrimitiveType.PrimitiveTypeName.FLOAT).named("temperature").named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group1 = factory.newGroup();
+ group1.add("temperature", 98.6f);
+
+ Group group2 = factory.newGroup();
+ group2.add("temperature", 37.0f);
+
+ return List.of(group1, group2);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ try (CloseableIterator iterator = reader.read(storageObject, null, 10)) {
+ assertTrue(iterator.hasNext());
+ Page page = iterator.next();
+
+ assertEquals(2, page.getPositionCount());
+
+ // Float is converted to double
+ assertEquals(98.6, ((DoubleBlock) page.getBlock(0)).getDouble(0), 0.1);
+ assertEquals(37.0, ((DoubleBlock) page.getBlock(0)).getDouble(1), 0.1);
+ }
+ }
+
+ public void testMetadataReturnsCorrectSourceType() throws Exception {
+ MessageType schema = Types.buildMessage().required(PrimitiveType.PrimitiveTypeName.INT64).named("id").named("test_schema");
+
+ byte[] parquetData = createParquetFile(schema, factory -> {
+ Group group = factory.newGroup();
+ group.add("id", 1L);
+ return List.of(group);
+ });
+
+ StorageObject storageObject = createStorageObject(parquetData);
+ ParquetFormatReader reader = new ParquetFormatReader(blockFactory);
+
+ SourceMetadata metadata = reader.metadata(storageObject);
+ assertEquals("parquet", metadata.sourceType());
+ }
+
+ @FunctionalInterface
+ private interface GroupCreator {
+ List create(SimpleGroupFactory factory);
+ }
+
+ private byte[] createParquetFile(MessageType schema, GroupCreator groupCreator) throws IOException {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+
+ OutputFile outputFile = new OutputFile() {
+ @Override
+ public PositionOutputStream create(long blockSizeHint) throws IOException {
+ return new PositionOutputStream() {
+ private long position = 0;
+
+ @Override
+ public long getPos() throws IOException {
+ return position;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ outputStream.write(b);
+ position++;
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ outputStream.write(b, off, len);
+ position += len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ outputStream.close();
+ }
+ };
+ }
+
+ @Override
+ public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+ return create(blockSizeHint);
+ }
+
+ @Override
+ public boolean supportsBlockSize() {
+ return false;
+ }
+
+ @Override
+ public long defaultBlockSize() {
+ return 0;
+ }
+
+ @Override
+ public String getPath() {
+ return "memory://test.parquet";
+ }
+ };
+
+ SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema);
+ List groups = groupCreator.create(groupFactory);
+
+ try (
+ ParquetWriter writer = ExampleParquetWriter.builder(outputFile)
+ .withType(schema)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .build()
+ ) {
+
+ for (Group group : groups) {
+ writer.write(group);
+ }
+ }
+
+ return outputStream.toByteArray();
+ }
+
+ private StorageObject createStorageObject(byte[] data) {
+ return new StorageObject() {
+ @Override
+ public InputStream newStream() throws IOException {
+ return new ByteArrayInputStream(data);
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ int pos = (int) position;
+ int len = (int) Math.min(length, data.length - position);
+ return new ByteArrayInputStream(data, pos, len);
+ }
+
+ @Override
+ public long length() throws IOException {
+ return data.length;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ return Instant.now();
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ return true;
+ }
+
+ @Override
+ public StoragePath path() {
+ return StoragePath.of("memory://test.parquet");
+ }
+ };
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapterTests.java b/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapterTests.java
new file mode 100644
index 0000000000000..456e83f3ff5e3
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-parquet/src/test/java/org/elasticsearch/xpack/esql/datasource/parquet/ParquetStorageObjectAdapterTests.java
@@ -0,0 +1,288 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.parquet;
+
+import org.apache.parquet.io.SeekableInputStream;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+
+public class ParquetStorageObjectAdapterTests extends ESTestCase {
+
+ public void testNullStorageObjectThrowsException() {
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ParquetStorageObjectAdapter(null));
+ assertEquals("storageObject cannot be null", e.getMessage());
+ }
+
+ public void testGetLength() throws IOException {
+ byte[] data = new byte[1024];
+ randomBytes(data);
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ assertEquals(1024, adapter.getLength());
+ }
+
+ public void testNewStreamReturnsSeekableInputStream() throws IOException {
+ byte[] data = new byte[100];
+ randomBytes(data);
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ assertNotNull(stream);
+ assertEquals(0, stream.getPos());
+ }
+ }
+
+ public void testSeekableInputStreamRead() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ assertEquals(1, stream.read());
+ assertEquals(1, stream.getPos());
+ assertEquals(2, stream.read());
+ assertEquals(2, stream.getPos());
+ }
+ }
+
+ public void testSeekableInputStreamReadArray() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ byte[] buffer = new byte[5];
+ int bytesRead = stream.read(buffer);
+ assertEquals(5, bytesRead);
+ assertEquals(5, stream.getPos());
+ assertArrayEquals(new byte[] { 1, 2, 3, 4, 5 }, buffer);
+ }
+ }
+
+ public void testSeekableInputStreamSeekForward() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ stream.seek(5);
+ assertEquals(5, stream.getPos());
+ assertEquals(6, stream.read());
+ assertEquals(6, stream.getPos());
+ }
+ }
+
+ public void testSeekableInputStreamSeekBackward() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createRangeReadStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ // Read some bytes to advance position
+ stream.read();
+ stream.read();
+ stream.read();
+ assertEquals(3, stream.getPos());
+
+ // Seek backward
+ stream.seek(1);
+ assertEquals(1, stream.getPos());
+ assertEquals(2, stream.read());
+ }
+ }
+
+ public void testSeekableInputStreamSeekToNegativePositionThrows() throws IOException {
+ byte[] data = new byte[100];
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ IOException e = expectThrows(IOException.class, () -> stream.seek(-1));
+ assertTrue(e.getMessage().contains("Cannot seek to negative position"));
+ }
+ }
+
+ public void testSeekableInputStreamSeekBeyondEndThrows() throws IOException {
+ byte[] data = new byte[100];
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ IOException e = expectThrows(IOException.class, () -> stream.seek(200));
+ assertTrue(e.getMessage().contains("Cannot seek beyond end of file"));
+ }
+ }
+
+ public void testSeekableInputStreamReadFully() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ byte[] buffer = new byte[5];
+ stream.readFully(buffer);
+ assertArrayEquals(new byte[] { 1, 2, 3, 4, 5 }, buffer);
+ assertEquals(5, stream.getPos());
+ }
+ }
+
+ public void testSeekableInputStreamReadFullyWithOffset() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ byte[] buffer = new byte[10];
+ stream.readFully(buffer, 2, 5);
+ assertArrayEquals(new byte[] { 0, 0, 1, 2, 3, 4, 5, 0, 0, 0 }, buffer);
+ assertEquals(5, stream.getPos());
+ }
+ }
+
+ public void testSeekableInputStreamReadByteBuffer() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ ByteBuffer buffer = ByteBuffer.allocate(5);
+ int bytesRead = stream.read(buffer);
+ assertEquals(5, bytesRead);
+ buffer.flip();
+ assertEquals(1, buffer.get());
+ assertEquals(2, buffer.get());
+ }
+ }
+
+ public void testSeekableInputStreamReadFullyByteBuffer() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ ByteBuffer buffer = ByteBuffer.allocate(5);
+ stream.readFully(buffer);
+ buffer.flip();
+ assertEquals(1, buffer.get());
+ assertEquals(2, buffer.get());
+ assertEquals(3, buffer.get());
+ assertEquals(4, buffer.get());
+ assertEquals(5, buffer.get());
+ }
+ }
+
+ public void testSeekableInputStreamSkip() throws IOException {
+ byte[] data = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ StorageObject storageObject = createStorageObject(data);
+
+ ParquetStorageObjectAdapter adapter = new ParquetStorageObjectAdapter(storageObject);
+
+ try (SeekableInputStream stream = adapter.newStream()) {
+ long skipped = stream.skip(3);
+ assertEquals(3, skipped);
+ assertEquals(3, stream.getPos());
+ assertEquals(4, stream.read());
+ }
+ }
+
+ private void randomBytes(byte[] data) {
+ random().nextBytes(data);
+ }
+
+ private StorageObject createStorageObject(byte[] data) {
+ return new StorageObject() {
+ @Override
+ public InputStream newStream() throws IOException {
+ return new ByteArrayInputStream(data);
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ // Simple implementation that doesn't support range reads
+ throw new UnsupportedOperationException("Range reads not supported in basic test");
+ }
+
+ @Override
+ public long length() throws IOException {
+ return data.length;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ return Instant.now();
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ return true;
+ }
+
+ @Override
+ public StoragePath path() {
+ return StoragePath.of("memory://test.parquet");
+ }
+ };
+ }
+
+ private StorageObject createRangeReadStorageObject(byte[] data) {
+ return new StorageObject() {
+ @Override
+ public InputStream newStream() throws IOException {
+ return new ByteArrayInputStream(data);
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ int pos = (int) position;
+ int len = (int) Math.min(length, data.length - position);
+ return new ByteArrayInputStream(data, pos, len);
+ }
+
+ @Override
+ public long length() throws IOException {
+ return data.length;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ return Instant.now();
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ return true;
+ }
+
+ @Override
+ public StoragePath path() {
+ return StoragePath.of("memory://test.parquet");
+ }
+ };
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-s3/README.md b/x-pack/plugin/esql-datasource-s3/README.md
new file mode 100644
index 0000000000000..d459ba74d6563
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/README.md
@@ -0,0 +1,140 @@
+# ESQL S3 Data Source Plugin
+
+This plugin provides AWS S3 storage support for ESQL external data sources.
+
+## Overview
+
+The S3 plugin enables ESQL to read data files directly from Amazon S3 buckets. It supports multiple S3 URI schemes and integrates with AWS authentication mechanisms.
+
+## Features
+
+- **S3 Storage Access** - Read files directly from S3 buckets
+- **Multiple URI Schemes** - Supports `s3://`, `s3a://`, and `s3n://` schemes
+- **Range Requests** - Efficient partial file reads for columnar formats
+- **AWS Authentication** - Supports IAM roles, access keys, and instance profiles
+
+## Usage
+
+Once installed, the plugin automatically registers the S3 storage provider. Use S3 URIs in ESQL queries:
+
+```sql
+FROM "s3://my-bucket/data/sales.parquet"
+| WHERE region = "EMEA"
+| STATS total = SUM(amount) BY product
+```
+
+```sql
+FROM "s3a://analytics-bucket/events/2024/01/events.csv"
+| KEEP timestamp, user_id, event_type
+| SORT timestamp DESC
+```
+
+### URI Schemes
+
+| Scheme | Description |
+|--------|-------------|
+| `s3://` | Standard S3 URI scheme |
+| `s3a://` | Hadoop S3A connector scheme (compatible) |
+| `s3n://` | Legacy Hadoop S3 native scheme (compatible) |
+
+## Configuration
+
+S3 access is configured via Elasticsearch settings or environment variables:
+
+### Environment Variables
+
+```bash
+AWS_ACCESS_KEY_ID=your-access-key
+AWS_SECRET_ACCESS_KEY=your-secret-key
+AWS_REGION=us-east-1
+```
+
+### IAM Roles
+
+When running on EC2 or EKS, the plugin automatically uses IAM roles attached to the instance or pod.
+
+## Dependencies
+
+This plugin bundles the AWS SDK v2:
+
+| Dependency | Version | Purpose |
+|------------|---------|---------|
+| software.amazon.awssdk:s3 | 2.x | S3 client |
+| software.amazon.awssdk:auth | 2.x | AWS authentication |
+| software.amazon.awssdk:sts | 2.x | STS for role assumption |
+| software.amazon.awssdk:apache-client | 2.x | HTTP client |
+| org.apache.httpcomponents:httpclient | 4.x | HTTP transport |
+
+## Architecture
+
+```
+┌─────────────────────────────────────────┐
+│ S3DataSourcePlugin │
+│ implements DataSourcePlugin │
+└─────────────────┬───────────────────────┘
+ │
+ │ provides
+ ▼
+┌─────────────────────────────────────────┐
+│ S3StorageProvider │
+│ implements StorageProvider │
+│ │
+│ - newObject(StoragePath) │
+│ - listObjects(StoragePath) │
+│ - exists(StoragePath) │
+│ - supportedSchemes() → [s3, s3a, s3n] │
+└─────────────────┬───────────────────────┘
+ │
+ │ creates
+ ▼
+┌─────────────────────────────────────────┐
+│ S3StorageObject │
+│ implements StorageObject │
+│ │
+│ - newStream() │
+│ - newStream(position, length) │
+│ - length() │
+│ - lastModified() │
+│ - exists() │
+└─────────────────────────────────────────┘
+```
+
+## Supported Operations
+
+| Operation | Description |
+|-----------|-------------|
+| `newObject()` | Create a reference to an S3 object |
+| `newStream()` | Read entire object as InputStream |
+| `newStream(pos, len)` | Read byte range (for columnar formats) |
+| `length()` | Get object size via HEAD request |
+| `lastModified()` | Get object modification time |
+| `exists()` | Check if object exists |
+| `listObjects()` | List objects with prefix |
+
+## Building
+
+```bash
+./gradlew :x-pack:plugin:esql-datasource-s3:build
+```
+
+## Testing
+
+```bash
+# Unit tests
+./gradlew :x-pack:plugin:esql-datasource-s3:test
+```
+
+## Security Considerations
+
+- Store AWS credentials securely using IAM roles or Elasticsearch keystore
+- Use VPC endpoints for private S3 access
+- Enable S3 bucket policies to restrict access
+- Consider using S3 Access Points for fine-grained access control
+
+## Installation
+
+The plugin is bundled with Elasticsearch and enabled by default when the ESQL feature is available.
+
+## License
+
+Elastic License 2.0
diff --git a/x-pack/plugin/esql-datasource-s3/build.gradle b/x-pack/plugin/esql-datasource-s3/build.gradle
new file mode 100644
index 0000000000000..3f0b5300cbcc0
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/build.gradle
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.publish'
+
+esplugin {
+ name = 'esql-datasource-s3'
+ description = 'S3 storage provider for ESQL external data sources'
+ classname = 'org.elasticsearch.xpack.esql.datasource.s3.S3DataSourcePlugin'
+ extendedPlugins = ['x-pack-esql']
+}
+
+base {
+ archivesName = 'esql-datasource-s3'
+}
+
+dependencies {
+ // SPI interfaces from ESQL core
+ compileOnly project(path: xpackModule('esql'))
+ compileOnly project(path: xpackModule('esql-core'))
+ compileOnly project(path: xpackModule('core'))
+ compileOnly project(':server')
+
+ // AWS SDK for S3 access - following repository-s3 pattern
+ // Using explicit module declarations instead of bundle for better classloading
+ implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:apache-client:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:url-connection-client:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:auth:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-core:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-xml-protocol:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:aws-json-protocol:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:regions:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:retries:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:s3:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:sts:${versions.awsv2sdk}"
+ implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}"
+
+ // Apache HTTP client for AWS SDK (required by apache-client module)
+ implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+
+ runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+ runtimeOnly "commons-logging:commons-logging:${versions.commonslogging}"
+ runtimeOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+ runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
+ runtimeOnly "software.amazon.awssdk:arns:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:json-utils:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}"
+ runtimeOnly "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}"
+
+ testImplementation project(':test:framework')
+ testImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.withType(org.elasticsearch.gradle.internal.AbstractDependenciesTask).configureEach {
+ // AWS SDK module mappings
+ mapping from: 'annotations', to: 'aws-sdk-2'
+ mapping from: 'apache-client', to: 'aws-sdk-2'
+ mapping from: 'arns', to: 'aws-sdk-2'
+ mapping from: 'auth', to: 'aws-sdk-2'
+ mapping from: 'aws-core', to: 'aws-sdk-2'
+ mapping from: 'aws-json-protocol', to: 'aws-sdk-2'
+ mapping from: 'aws-query-protocol', to: 'aws-sdk-2'
+ mapping from: 'aws-xml-protocol', to: 'aws-sdk-2'
+ mapping from: 'checksums', to: 'aws-sdk-2'
+ mapping from: 'checksums-spi', to: 'aws-sdk-2'
+ mapping from: 'endpoints-spi', to: 'aws-sdk-2'
+ mapping from: 'http-auth', to: 'aws-sdk-2'
+ mapping from: 'http-auth-aws', to: 'aws-sdk-2'
+ mapping from: 'http-auth-spi', to: 'aws-sdk-2'
+ mapping from: 'http-client-spi', to: 'aws-sdk-2'
+ mapping from: 'identity-spi', to: 'aws-sdk-2'
+ mapping from: 'json-utils', to: 'aws-sdk-2'
+ mapping from: 'metrics-spi', to: 'aws-sdk-2'
+ mapping from: 'profiles', to: 'aws-sdk-2'
+ mapping from: 'protocol-core', to: 'aws-sdk-2'
+ mapping from: 'regions', to: 'aws-sdk-2'
+ mapping from: 'retries', to: 'aws-sdk-2'
+ mapping from: 'retries-spi', to: 'aws-sdk-2'
+ mapping from: 's3', to: 'aws-sdk-2'
+ mapping from: 'sdk-core', to: 'aws-sdk-2'
+ mapping from: 'sts', to: 'aws-sdk-2'
+ mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
+ mapping from: 'url-connection-client', to: 'aws-sdk-2'
+ mapping from: 'utils', to: 'aws-sdk-2'
+}
+
+tasks.named("thirdPartyAudit").configure {
+ ignoreMissingClasses(
+ // missing/unused classes from commons-logging (used by Apache HTTP client)
+ 'javax.servlet.ServletContextEvent',
+ 'javax.servlet.ServletContextListener',
+ 'org.apache.avalon.framework.logger.Logger',
+ 'org.apache.log.Hierarchy',
+ 'org.apache.log.Logger',
+
+ // We use the Apache HTTP client rather than AWS CRT, so these classes are not needed
+ 'software.amazon.awssdk.crt.CRT',
+ 'software.amazon.awssdk.crt.auth.credentials.Credentials',
+ 'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
+ 'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigner',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignatureType',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignedBodyHeaderType',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSigningAlgorithm',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig',
+ 'software.amazon.awssdk.crt.auth.signing.AwsSigningResult',
+ 'software.amazon.awssdk.crt.http.HttpHeader',
+ 'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
+ 'software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting$HttpProxyEnvironmentVariableType',
+ 'software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting',
+ 'software.amazon.awssdk.crt.http.HttpProxyOptions',
+ 'software.amazon.awssdk.crt.http.HttpRequest',
+ 'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
+ 'software.amazon.awssdk.crt.io.ClientBootstrap',
+ 'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
+ 'software.amazon.awssdk.crt.io.StandardRetryOptions',
+ 'software.amazon.awssdk.crt.io.TlsCipherPreference',
+ 'software.amazon.awssdk.crt.io.TlsContext',
+ 'software.amazon.awssdk.crt.io.TlsContextOptions',
+ 'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
+ 'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
+ 'software.amazon.awssdk.crt.s3.ChecksumConfig',
+ 'software.amazon.awssdk.crt.s3.ResumeToken',
+ 'software.amazon.awssdk.crt.s3.S3Client',
+ 'software.amazon.awssdk.crt.s3.S3ClientOptions',
+ 'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
+ 'software.amazon.awssdk.crt.s3.S3MetaRequest',
+ 'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
+ 'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
+ 'software.amazon.awssdk.crt.s3.S3MetaRequestProgress',
+ 'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
+ 'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
+ 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
+ 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
+ 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
+ 'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
+ 'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
+ 'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
+
+ // We don't use eventstream-based features
+ 'software.amazon.eventstream.HeaderValue',
+ 'software.amazon.eventstream.Message',
+ 'software.amazon.eventstream.MessageDecoder'
+ )
+}
diff --git a/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-LICENSE.txt b/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-LICENSE.txt
new file mode 100644
index 0000000000000..1eef70a9b9f42
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-LICENSE.txt
@@ -0,0 +1,206 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Note: Other license terms may apply to certain, identified software files contained within or distributed
+ with the accompanying software if such terms are included in the directory containing the accompanying software.
+ Such other license terms will then apply in lieu of the terms of the software license above.
diff --git a/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-NOTICE.txt b/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-NOTICE.txt
new file mode 100644
index 0000000000000..f3c4db7d1724e
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/licenses/aws-sdk-2-NOTICE.txt
@@ -0,0 +1,26 @@
+AWS SDK for Java 2.0
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+This product includes software developed by
+Amazon Technologies, Inc (http://www.amazon.com/).
+
+**********************
+THIRD PARTY COMPONENTS
+**********************
+This software includes third party software subject to the following copyrights:
+- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+- Apache Commons Lang - https://github.com/apache/commons-lang
+- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams
+- Jackson-core - https://github.com/FasterXML/jackson-core
+- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
+
+The licenses for these third party components are included in LICENSE.txt
+
+- For Apache Commons Lang see also this required NOTICE:
+ Apache Commons Lang
+ Copyright 2001-2020 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (https://www.apache.org/).
+
diff --git a/x-pack/plugin/esql-datasource-s3/licenses/reactive-streams-LICENSE.txt b/x-pack/plugin/esql-datasource-s3/licenses/reactive-streams-LICENSE.txt
new file mode 100644
index 0000000000000..1e141c13ddba2
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/licenses/reactive-streams-LICENSE.txt
@@ -0,0 +1,7 @@
+MIT No Attribution
+
+Copyright 2014 Reactive Streams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/x-pack/plugin/esql-datasource-s3/licenses/reactive-streams-NOTICE.txt b/x-pack/plugin/esql-datasource-s3/licenses/reactive-streams-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3Configuration.java b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3Configuration.java
new file mode 100644
index 0000000000000..58f855497e33d
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3Configuration.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasource.s3;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Configuration for S3 access including credentials and endpoint settings.
+ */
+public class S3Configuration {
+
+ private final String accessKey;
+ private final String secretKey;
+ private final String endpoint;
+ private final String region;
+
+ private S3Configuration(String accessKey, String secretKey, String endpoint, String region) {
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+ this.endpoint = endpoint;
+ this.region = region;
+ }
+
+ public static S3Configuration fromParams(Map params) {
+ if (params == null || params.isEmpty()) {
+ return null;
+ }
+
+ String accessKey = extractStringParam(params, "access_key");
+ String secretKey = extractStringParam(params, "secret_key");
+ String endpoint = extractStringParam(params, "endpoint");
+ String region = extractStringParam(params, "region");
+
+ if (accessKey == null && secretKey == null && endpoint == null && region == null) {
+ return null;
+ }
+
+ return new S3Configuration(accessKey, secretKey, endpoint, region);
+ }
+
+ public static S3Configuration fromFields(String accessKey, String secretKey, String endpoint, String region) {
+ if (accessKey == null && secretKey == null && endpoint == null && region == null) {
+ return null;
+ }
+ return new S3Configuration(accessKey, secretKey, endpoint, region);
+ }
+
+ private static String extractStringParam(Map params, String key) {
+ Expression expr = params.get(key);
+ if (expr instanceof org.elasticsearch.xpack.esql.core.expression.Literal literal) {
+ Object value = literal.value();
+ if (value instanceof BytesRef bytesRef) {
+ return BytesRefs.toString(bytesRef);
+ }
+ return value != null ? value.toString() : null;
+ }
+ return null;
+ }
+
+ public String accessKey() {
+ return accessKey;
+ }
+
+ public String secretKey() {
+ return secretKey;
+ }
+
+ public String endpoint() {
+ return endpoint;
+ }
+
+ public String region() {
+ return region;
+ }
+
+ public boolean hasCredentials() {
+ return accessKey != null && secretKey != null;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ S3Configuration that = (S3Configuration) o;
+ return Objects.equals(accessKey, that.accessKey)
+ && Objects.equals(secretKey, that.secretKey)
+ && Objects.equals(endpoint, that.endpoint)
+ && Objects.equals(region, that.region);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(accessKey, secretKey, endpoint, region);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3DataSourcePlugin.java b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3DataSourcePlugin.java
new file mode 100644
index 0000000000000..ea4c35026f09a
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3DataSourcePlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.s3;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProvider;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProviderFactory;
+
+import java.util.Map;
+
+/**
+ * Data source plugin providing S3 storage support for ESQL.
+ * Supports s3://, s3a://, and s3n:// URI schemes.
+ */
+public class S3DataSourcePlugin extends Plugin implements DataSourcePlugin {
+
+ @Override
+ public Map storageProviders(Settings settings) {
+ StorageProviderFactory s3Factory = new StorageProviderFactory() {
+ @Override
+ public StorageProvider create(Settings settings) {
+ return new S3StorageProvider(null);
+ }
+
+ @Override
+ public StorageProvider create(Settings settings, Map config) {
+ if (config == null || config.isEmpty()) {
+ return create(settings);
+ }
+ S3Configuration s3Config = S3Configuration.fromFields(
+ (String) config.get("access_key"),
+ (String) config.get("secret_key"),
+ (String) config.get("endpoint"),
+ (String) config.get("region")
+ );
+ return new S3StorageProvider(s3Config);
+ }
+ };
+ return Map.of("s3", s3Factory, "s3a", s3Factory, "s3n", s3Factory);
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageObject.java b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageObject.java
new file mode 100644
index 0000000000000..8d98ffeaa7fda
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageObject.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.s3;
+
+import software.amazon.awssdk.core.ResponseInputStream;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.util.concurrent.Executor;
+
+/**
+ * StorageObject implementation for S3 using AWS SDK v2.
+ * Supports full and range reads, metadata retrieval, and optional native async via S3AsyncClient.
+ */
+public final class S3StorageObject implements StorageObject {
+ private final S3Client s3Client;
+ private final S3AsyncClient s3AsyncClient;
+ private final String bucket;
+ private final String key;
+ private final StoragePath path;
+
+ private Long cachedLength;
+ private Instant cachedLastModified;
+ private Boolean cachedExists;
+
+ public S3StorageObject(S3Client s3Client, String bucket, String key, StoragePath path) {
+ this(s3Client, null, bucket, key, path);
+ }
+
+ public S3StorageObject(S3Client s3Client, S3AsyncClient s3AsyncClient, String bucket, String key, StoragePath path) {
+ if (s3Client == null) {
+ throw new IllegalArgumentException("s3Client cannot be null");
+ }
+ if (bucket == null || bucket.isEmpty()) {
+ throw new IllegalArgumentException("bucket cannot be null or empty");
+ }
+ if (key == null) {
+ throw new IllegalArgumentException("key cannot be null");
+ }
+ if (path == null) {
+ throw new IllegalArgumentException("path cannot be null");
+ }
+ this.s3Client = s3Client;
+ this.s3AsyncClient = s3AsyncClient;
+ this.bucket = bucket;
+ this.key = key;
+ this.path = path;
+ }
+
+ public S3StorageObject(S3Client s3Client, String bucket, String key, StoragePath path, long length) {
+ this(s3Client, bucket, key, path);
+ this.cachedLength = length;
+ }
+
+ public S3StorageObject(S3Client s3Client, S3AsyncClient s3AsyncClient, String bucket, String key, StoragePath path, long length) {
+ this(s3Client, s3AsyncClient, bucket, key, path);
+ this.cachedLength = length;
+ }
+
+ public S3StorageObject(S3Client s3Client, String bucket, String key, StoragePath path, long length, Instant lastModified) {
+ this(s3Client, bucket, key, path, length);
+ this.cachedLastModified = lastModified;
+ }
+
+ public S3StorageObject(
+ S3Client s3Client,
+ S3AsyncClient s3AsyncClient,
+ String bucket,
+ String key,
+ StoragePath path,
+ long length,
+ Instant lastModified
+ ) {
+ this(s3Client, s3AsyncClient, bucket, key, path, length);
+ this.cachedLastModified = lastModified;
+ }
+
+ @Override
+ public InputStream newStream() throws IOException {
+ try {
+ GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build();
+ ResponseInputStream response = s3Client.getObject(request);
+
+ if (cachedLength == null) {
+ cachedLength = response.response().contentLength();
+ }
+ if (cachedLastModified == null) {
+ cachedLastModified = response.response().lastModified();
+ }
+
+ return response;
+ } catch (NoSuchKeyException e) {
+ throw new IOException("Object not found: " + path, e);
+ } catch (Exception e) {
+ throw new IOException("Failed to read object from " + path, e);
+ }
+ }
+
+ @Override
+ public InputStream newStream(long position, long length) throws IOException {
+ if (position < 0) {
+ throw new IllegalArgumentException("position must be non-negative, got: " + position);
+ }
+ if (length < 0) {
+ throw new IllegalArgumentException("length must be non-negative, got: " + length);
+ }
+
+ long endPosition = position + length - 1;
+ String rangeHeader = Strings.format("bytes=%d-%d", position, endPosition);
+
+ try {
+ GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).range(rangeHeader).build();
+ ResponseInputStream response = s3Client.getObject(request);
+
+ if (cachedLength == null && response.response().contentLength() != null) {
+ String contentRange = response.response().contentRange();
+ if (contentRange != null && contentRange.contains("/")) {
+ String[] parts = contentRange.split("/");
+ if (parts.length == 2 && parts[1].equals("*") == false) {
+ try {
+ cachedLength = Long.parseLong(parts[1]);
+ } catch (NumberFormatException ignored) {}
+ }
+ }
+ }
+ if (cachedLastModified == null) {
+ cachedLastModified = response.response().lastModified();
+ }
+
+ return response;
+ } catch (NoSuchKeyException e) {
+ throw new IOException("Object not found: " + path, e);
+ } catch (Exception e) {
+ throw new IOException("Range request failed for " + path, e);
+ }
+ }
+
+ @Override
+ public long length() throws IOException {
+ if (cachedLength == null) {
+ fetchMetadata();
+ }
+ if (cachedExists != null && cachedExists == false) {
+ throw new IOException("Object not found: " + path);
+ }
+ return cachedLength;
+ }
+
+ @Override
+ public Instant lastModified() throws IOException {
+ if (cachedLastModified == null) {
+ fetchMetadata();
+ }
+ return cachedLastModified;
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ if (cachedExists == null) {
+ fetchMetadata();
+ }
+ return cachedExists;
+ }
+
+ @Override
+ public StoragePath path() {
+ return path;
+ }
+
+ private void fetchMetadata() throws IOException {
+ try {
+ HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(key).build();
+ HeadObjectResponse response = s3Client.headObject(request);
+
+ cachedExists = true;
+ cachedLength = response.contentLength();
+ cachedLastModified = response.lastModified();
+ } catch (NoSuchKeyException e) {
+ cachedExists = false;
+ cachedLength = 0L;
+ cachedLastModified = null;
+ } catch (Exception e) {
+ throw new IOException("HeadObject request failed for " + path, e);
+ }
+ }
+
+ public String bucket() {
+ return bucket;
+ }
+
+ public String key() {
+ return key;
+ }
+
+ @Override
+ public void readBytesAsync(long position, long length, Executor executor, ActionListener listener) {
+ if (s3AsyncClient == null) {
+ StorageObject.super.readBytesAsync(position, length, executor, listener);
+ return;
+ }
+
+ if (position < 0) {
+ listener.onFailure(new IllegalArgumentException("position must be non-negative, got: " + position));
+ return;
+ }
+ if (length < 0) {
+ listener.onFailure(new IllegalArgumentException("length must be non-negative, got: " + length));
+ return;
+ }
+
+ long endPosition = position + length - 1;
+ String rangeHeader = Strings.format("bytes=%d-%d", position, endPosition);
+
+ GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).range(rangeHeader).build();
+
+ s3AsyncClient.getObject(request, AsyncResponseTransformer.toBytes()).whenComplete((responseBytes, throwable) -> {
+ if (throwable != null) {
+ Throwable cause = throwable.getCause() != null ? throwable.getCause() : throwable;
+ if (cause instanceof NoSuchKeyException) {
+ listener.onFailure(new IOException("Object not found: " + path, cause));
+ } else {
+ listener.onFailure(cause instanceof Exception ex ? ex : new RuntimeException(cause));
+ }
+ return;
+ }
+
+ GetObjectResponse response = responseBytes.response();
+ if (cachedLastModified == null) {
+ cachedLastModified = response.lastModified();
+ }
+ if (cachedLength == null) {
+ String contentRange = response.contentRange();
+ if (contentRange != null && contentRange.contains("/")) {
+ String[] parts = contentRange.split("/");
+ if (parts.length == 2 && parts[1].equals("*") == false) {
+ try {
+ cachedLength = Long.parseLong(parts[1]);
+ } catch (NumberFormatException ignored) {}
+ }
+ }
+ }
+
+ listener.onResponse(ByteBuffer.wrap(responseBytes.asByteArray()));
+ });
+ }
+
+ @Override
+ public boolean supportsNativeAsync() {
+ return s3AsyncClient != null;
+ }
+
+ @Override
+ public String toString() {
+ return "S3StorageObject{bucket=" + bucket + ", key=" + key + ", path=" + path + "}";
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageProvider.java b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageProvider.java
new file mode 100644
index 0000000000000..78dcd1a90e77a
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/java/org/elasticsearch/xpack/esql/datasource/s3/S3StorageProvider.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.datasource.s3;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import org.elasticsearch.xpack.esql.datasources.StorageEntry;
+import org.elasticsearch.xpack.esql.datasources.StorageIterator;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
+import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
+import org.elasticsearch.xpack.esql.datasources.spi.StorageProvider;
+
+import java.io.IOException;
+import java.net.URI;
+import java.time.Instant;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+/**
+ * StorageProvider implementation for S3 using AWS SDK v2.
+ */
+public final class S3StorageProvider implements StorageProvider {
+ private final S3Client s3Client;
+ private final S3Configuration config;
+
+ public S3StorageProvider(S3Configuration config) {
+ this.config = config;
+ this.s3Client = buildS3Client(config);
+ }
+
+ private static S3Client buildS3Client(S3Configuration config) {
+ S3ClientBuilder builder = S3Client.builder();
+
+ AwsCredentialsProvider credentialsProvider;
+ if (config != null && config.hasCredentials()) {
+ credentialsProvider = StaticCredentialsProvider.create(AwsBasicCredentials.create(config.accessKey(), config.secretKey()));
+ } else {
+ credentialsProvider = DefaultCredentialsProvider.create();
+ }
+ builder.credentialsProvider(credentialsProvider);
+
+ if (config != null && config.region() != null) {
+ builder.region(Region.of(config.region()));
+ } else {
+ builder.region(Region.US_EAST_1);
+ }
+
+ if (config != null && config.endpoint() != null) {
+ builder.endpointOverride(URI.create(config.endpoint()));
+ builder.forcePathStyle(true);
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path) {
+ validateS3Scheme(path);
+ String bucket = path.host();
+ String key = extractKey(path);
+ return new S3StorageObject(s3Client, bucket, key, path);
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length) {
+ validateS3Scheme(path);
+ String bucket = path.host();
+ String key = extractKey(path);
+ return new S3StorageObject(s3Client, bucket, key, path, length);
+ }
+
+ @Override
+ public StorageObject newObject(StoragePath path, long length, Instant lastModified) {
+ validateS3Scheme(path);
+ String bucket = path.host();
+ String key = extractKey(path);
+ return new S3StorageObject(s3Client, bucket, key, path, length, lastModified);
+ }
+
+ @Override
+ public StorageIterator listObjects(StoragePath prefix, boolean recursive) throws IOException {
+ validateS3Scheme(prefix);
+ String bucket = prefix.host();
+ String keyPrefix = extractKey(prefix);
+
+ if (keyPrefix.isEmpty() == false && keyPrefix.endsWith(StoragePath.PATH_SEPARATOR) == false) {
+ keyPrefix += StoragePath.PATH_SEPARATOR;
+ }
+
+ // S3 is a flat namespace — ListObjectsV2 is inherently prefix-based and recursive.
+ // The recursive flag is effectively ignored.
+ return new S3StorageIterator(s3Client, bucket, keyPrefix, prefix);
+ }
+
+ @Override
+ public boolean exists(StoragePath path) throws IOException {
+ validateS3Scheme(path);
+ String bucket = path.host();
+ String key = extractKey(path);
+
+ try {
+ HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(key).build();
+ s3Client.headObject(request);
+ return true;
+ } catch (NoSuchKeyException e) {
+ return false;
+ } catch (Exception e) {
+ throw new IOException("Failed to check existence of " + path, e);
+ }
+ }
+
+ @Override
+ public List supportedSchemes() {
+ return List.of("s3", "s3a", "s3n");
+ }
+
+ @Override
+ public void close() throws IOException {
+ s3Client.close();
+ }
+
+ private void validateS3Scheme(StoragePath path) {
+ String scheme = path.scheme().toLowerCase(Locale.ROOT);
+ if (scheme.equals("s3") == false && scheme.equals("s3a") == false && scheme.equals("s3n") == false) {
+ throw new IllegalArgumentException("S3StorageProvider only supports s3://, s3a://, and s3n:// schemes, got: " + scheme);
+ }
+ }
+
+ private String extractKey(StoragePath path) {
+ String key = path.path();
+ if (key.startsWith(StoragePath.PATH_SEPARATOR)) {
+ key = key.substring(1);
+ }
+ return key;
+ }
+
+ public S3Client s3Client() {
+ return s3Client;
+ }
+
+ public S3Configuration config() {
+ return config;
+ }
+
+ @Override
+ public String toString() {
+ return "S3StorageProvider{config=" + config + "}";
+ }
+
+ /**
+ * Iterator for S3 object listing with pagination support.
+ */
+ private static final class S3StorageIterator implements StorageIterator {
+ private final S3Client s3Client;
+ private final String bucket;
+ private final String prefix;
+ private final StoragePath baseDirectory;
+
+ private Iterator currentBatch;
+ private String continuationToken;
+ private boolean hasMorePages;
+ private boolean initialized;
+
+ S3StorageIterator(S3Client s3Client, String bucket, String prefix, StoragePath baseDirectory) {
+ this.s3Client = s3Client;
+ this.bucket = bucket;
+ this.prefix = prefix;
+ this.baseDirectory = baseDirectory;
+ this.hasMorePages = true;
+ this.initialized = false;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (initialized == false) {
+ fetchNextBatch();
+ initialized = true;
+ }
+
+ if (currentBatch != null && currentBatch.hasNext()) {
+ return true;
+ }
+
+ if (hasMorePages) {
+ fetchNextBatch();
+ return currentBatch != null && currentBatch.hasNext();
+ }
+
+ return false;
+ }
+
+ @Override
+ public StorageEntry next() {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+
+ S3Object s3Object = currentBatch.next();
+ String fullPath = baseDirectory.scheme() + StoragePath.SCHEME_SEPARATOR + bucket + StoragePath.PATH_SEPARATOR + s3Object.key();
+ StoragePath objectPath = StoragePath.of(fullPath);
+
+ return new StorageEntry(objectPath, s3Object.size(), s3Object.lastModified());
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No resources to close
+ }
+
+ private void fetchNextBatch() {
+ try {
+ ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder().bucket(bucket).prefix(prefix);
+
+ if (continuationToken != null) {
+ requestBuilder.continuationToken(continuationToken);
+ }
+
+ ListObjectsV2Response response = s3Client.listObjectsV2(requestBuilder.build());
+
+ currentBatch = response.contents().iterator();
+ continuationToken = response.nextContinuationToken();
+ hasMorePages = response.isTruncated();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to list objects in bucket " + bucket + " with prefix " + prefix, e);
+ }
+ }
+ }
+}
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/plugin-metadata/entitlement-policy.yaml b/x-pack/plugin/esql-datasource-s3/src/main/plugin-metadata/entitlement-policy.yaml
new file mode 100644
index 0000000000000..394e5e38d9f59
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/plugin-metadata/entitlement-policy.yaml
@@ -0,0 +1,3 @@
+ALL-UNNAMED:
+ - manage_threads
+ - outbound_network
diff --git a/x-pack/plugin/esql-datasource-s3/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin b/x-pack/plugin/esql-datasource-s3/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
new file mode 100644
index 0000000000000..331dff3bd0043
--- /dev/null
+++ b/x-pack/plugin/esql-datasource-s3/src/main/resources/META-INF/services/org.elasticsearch.xpack.esql.datasources.spi.DataSourcePlugin
@@ -0,0 +1 @@
+org.elasticsearch.xpack.esql.datasource.s3.S3DataSourcePlugin
diff --git a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverter.java b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverter.java
new file mode 100644
index 0000000000000..db5170c74e20c
--- /dev/null
+++ b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverter.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.arrow;
+
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.Types;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+
+/**
+ * Converts Apache Arrow FieldVector to ESQL Blocks.
+ * This is the inverse operation of {@link BlockConverter} (Block → Arrow).
+ * Together they provide symmetric conversion: Block ↔ Arrow.
+ *
+ * Type Mapping (symmetric with BlockConverter):
+ *
+ * Arrow FLOAT4 (Float4Vector) → ESQL double (DoubleBlock) - {@link FromFloat32} (ESQL maps FLOAT to DOUBLE)
+ * Arrow FLOAT8 (Float8Vector) ↔ ESQL double (DoubleBlock) - {@link FromFloat64} / {@link BlockConverter.AsFloat64}
+ * Arrow BIGINT (BigIntVector) ↔ ESQL long (LongBlock) - {@link FromInt64} / {@link BlockConverter.AsInt64}
+ * Arrow INT (IntVector) ↔ ESQL integer (IntBlock) - {@link FromInt32} / {@link BlockConverter.AsInt32}
+ * Arrow BIT (BitVector) ↔ ESQL boolean (BooleanBlock) - {@link FromBoolean} / {@link BlockConverter.AsBoolean}
+ * Arrow VARCHAR (VarCharVector) ↔ ESQL keyword (BytesRefBlock) - {@link FromVarChar} / {@link BlockConverter.AsVarChar}
+ * Arrow VARBINARY (VarBinaryVector) ↔ ESQL ip/binary (BytesRefBlock) -
+ * {@link FromVarBinary} / {@link BlockConverter.AsVarBinary}
+ * Arrow TIMESTAMPMICRO (TimeStampMicroVector) → ESQL datetime (LongBlock) - {@link FromTimestampMicro}
+ * Arrow TIMESTAMPMICROTZ (TimeStampMicroTZVector) → ESQL datetime (LongBlock) - {@link FromTimestampMicroTZ}
+ *
+ *
+ * Note: Timestamp types convert from microseconds (Arrow) to milliseconds (ESQL).
+ * Float types (FLOAT4) are converted to double (ESQL doesn't have a separate float type).
+ *
+ *
This converter is designed to be used in the arrow module to keep Arrow dependencies isolated,
+ * preventing Arrow from leaking into the compute module.
+ */
+public abstract class ArrowToBlockConverter {
+
+ /**
+ * Convert an Arrow FieldVector to an ESQL Block.
+ * @param vector the Arrow vector
+ * @param factory the block factory for memory management
+ * @return the ESQL block
+ */
+ public abstract Block convert(FieldVector vector, BlockFactory factory);
+
+ /**
+ * Create a converter for the given Arrow type.
+ * @param arrowType the Arrow minor type
+ * @return the appropriate converter, or null if the type is not supported
+ */
+ public static ArrowToBlockConverter forType(Types.MinorType arrowType) {
+ return switch (arrowType) {
+ case FLOAT4 -> new FromFloat32();
+ case FLOAT8 -> new FromFloat64();
+ case BIGINT -> new FromInt64();
+ case INT -> new FromInt32();
+ case BIT -> new FromBoolean();
+ case VARCHAR -> new FromVarChar();
+ case VARBINARY -> new FromVarBinary();
+ case TIMESTAMPMICRO -> new FromTimestampMicro();
+ case TIMESTAMPMICROTZ -> new FromTimestampMicroTZ();
+ default -> null;
+ };
+ }
+
+ /**
+ * Conversion from Arrow Float4Vector (float) to ESQL DoubleBlock.
+ * ESQL maps FLOAT to DOUBLE, so we convert float32 to double.
+ */
+ public static class FromFloat32 extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ Float4Vector f4v = (Float4Vector) vector;
+ int valueCount = f4v.getValueCount();
+
+ try (DoubleBlock.Builder builder = factory.newDoubleBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (f4v.isNull(i)) {
+ builder.appendNull();
+ } else {
+ // Convert float to double for ESQL
+ builder.appendDouble((double) f4v.get(i));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow Float8Vector (double) to ESQL DoubleBlock.
+ * Symmetric with {@link BlockConverter.AsFloat64}.
+ */
+ public static class FromFloat64 extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ Float8Vector f8v = (Float8Vector) vector;
+ int valueCount = f8v.getValueCount();
+
+ try (DoubleBlock.Builder builder = factory.newDoubleBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (f8v.isNull(i)) {
+ builder.appendNull();
+ } else {
+ builder.appendDouble(f8v.get(i));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow BigIntVector (long) to ESQL LongBlock.
+ * Symmetric with {@link BlockConverter.AsInt64}.
+ */
+ public static class FromInt64 extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ BigIntVector bigIntVector = (BigIntVector) vector;
+ int valueCount = bigIntVector.getValueCount();
+
+ try (LongBlock.Builder builder = factory.newLongBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (bigIntVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ builder.appendLong(bigIntVector.get(i));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow IntVector (int) to ESQL IntBlock.
+ * Symmetric with {@link BlockConverter.AsInt32}.
+ */
+ public static class FromInt32 extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ IntVector intVector = (IntVector) vector;
+ int valueCount = intVector.getValueCount();
+
+ try (IntBlock.Builder builder = factory.newIntBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (intVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ builder.appendInt(intVector.get(i));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow BitVector (boolean) to ESQL BooleanBlock.
+ * Symmetric with {@link BlockConverter.AsBoolean}.
+ */
+ public static class FromBoolean extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ BitVector bitVector = (BitVector) vector;
+ int valueCount = bitVector.getValueCount();
+
+ try (BooleanBlock.Builder builder = factory.newBooleanBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (bitVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ builder.appendBoolean(bitVector.get(i) != 0);
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow VarCharVector (string) to ESQL BytesRefBlock.
+ * Symmetric with {@link BlockConverter.AsVarChar}.
+ */
+ public static class FromVarChar extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ VarCharVector varCharVector = (VarCharVector) vector;
+ int valueCount = varCharVector.getValueCount();
+
+ try (BytesRefBlock.Builder builder = factory.newBytesRefBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (varCharVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ byte[] bytes = varCharVector.get(i);
+ builder.appendBytesRef(new BytesRef(bytes));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow VarBinaryVector (binary) to ESQL BytesRefBlock.
+ * Symmetric with {@link BlockConverter.AsVarBinary}.
+ */
+ public static class FromVarBinary extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ VarBinaryVector varBinaryVector = (VarBinaryVector) vector;
+ int valueCount = varBinaryVector.getValueCount();
+
+ try (BytesRefBlock.Builder builder = factory.newBytesRefBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (varBinaryVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ byte[] bytes = varBinaryVector.get(i);
+ builder.appendBytesRef(new BytesRef(bytes));
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow TimeStampMicroVector (timestamp without timezone, microseconds) to ESQL LongBlock.
+ * Arrow stores timestamps as microseconds since epoch; ESQL stores datetime as milliseconds.
+ */
+ public static class FromTimestampMicro extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ TimeStampMicroVector tsVector = (TimeStampMicroVector) vector;
+ int valueCount = tsVector.getValueCount();
+
+ try (LongBlock.Builder builder = factory.newLongBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (tsVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ // Convert from microseconds to milliseconds
+ long micros = tsVector.get(i);
+ builder.appendLong(micros / 1000);
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ /**
+ * Conversion from Arrow TimeStampMicroTZVector (timestamp with timezone, microseconds) to ESQL LongBlock.
+ * Arrow stores timestamps as microseconds since epoch; ESQL stores datetime as milliseconds.
+ * The timezone information is not preserved in ESQL's datetime type.
+ */
+ public static class FromTimestampMicroTZ extends ArrowToBlockConverter {
+ @Override
+ public Block convert(FieldVector vector, BlockFactory factory) {
+ TimeStampMicroTZVector tsVector = (TimeStampMicroTZVector) vector;
+ int valueCount = tsVector.getValueCount();
+
+ try (LongBlock.Builder builder = factory.newLongBlockBuilder(valueCount)) {
+ for (int i = 0; i < valueCount; i++) {
+ if (tsVector.isNull(i)) {
+ builder.appendNull();
+ } else {
+ // Convert from microseconds to milliseconds
+ long micros = tsVector.get(i);
+ builder.appendLong(micros / 1000);
+ }
+ }
+ return builder.build();
+ }
+ }
+ }
+}
diff --git a/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverterTests.java b/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverterTests.java
new file mode 100644
index 0000000000000..378c7af3dddfa
--- /dev/null
+++ b/x-pack/plugin/esql/arrow/src/test/java/org/elasticsearch/xpack/esql/arrow/ArrowToBlockConverterTests.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.arrow;
+
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.Types;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.charset.StandardCharsets;
+
+public class ArrowToBlockConverterTests extends ESTestCase {
+
+ private RootAllocator allocator;
+ private BlockFactory blockFactory;
+
+ @Before
+ public void setup() {
+ allocator = new RootAllocator();
+ blockFactory = BlockFactory.getInstance(new NoopCircuitBreaker("test-noop"), BigArrays.NON_RECYCLING_INSTANCE);
+ }
+
+ @After
+ public void cleanup() {
+ allocator.close();
+ }
+
+ public void testFromFloat64() {
+ try (Float8Vector vector = new Float8Vector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, 1.5);
+ vector.set(1, 2.5);
+ vector.setNull(2);
+ vector.set(3, 3.5);
+ vector.set(4, 4.5);
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromFloat64();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof DoubleBlock);
+ DoubleBlock doubleBlock = (DoubleBlock) block;
+
+ assertEquals(5, doubleBlock.getPositionCount());
+ assertEquals(1.5, doubleBlock.getDouble(0), 0.0);
+ assertEquals(2.5, doubleBlock.getDouble(1), 0.0);
+ assertTrue(doubleBlock.isNull(2));
+ assertEquals(3.5, doubleBlock.getDouble(3), 0.0);
+ assertEquals(4.5, doubleBlock.getDouble(4), 0.0);
+ }
+ }
+ }
+
+ public void testFromFloat64AllNulls() {
+ try (Float8Vector vector = new Float8Vector("test", allocator)) {
+ vector.allocateNew(3);
+ vector.setNull(0);
+ vector.setNull(1);
+ vector.setNull(2);
+ vector.setValueCount(3);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromFloat64();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof DoubleBlock);
+ DoubleBlock doubleBlock = (DoubleBlock) block;
+
+ assertEquals(3, doubleBlock.getPositionCount());
+ assertTrue(doubleBlock.isNull(0));
+ assertTrue(doubleBlock.isNull(1));
+ assertTrue(doubleBlock.isNull(2));
+ }
+ }
+ }
+
+ public void testFromInt64() {
+ try (BigIntVector vector = new BigIntVector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, 100L);
+ vector.set(1, 200L);
+ vector.setNull(2);
+ vector.set(3, 300L);
+ vector.set(4, 400L);
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromInt64();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof LongBlock);
+ LongBlock longBlock = (LongBlock) block;
+
+ assertEquals(5, longBlock.getPositionCount());
+ assertEquals(100L, longBlock.getLong(0));
+ assertEquals(200L, longBlock.getLong(1));
+ assertTrue(longBlock.isNull(2));
+ assertEquals(300L, longBlock.getLong(3));
+ assertEquals(400L, longBlock.getLong(4));
+ }
+ }
+ }
+
+ public void testFromInt32() {
+ try (IntVector vector = new IntVector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, 10);
+ vector.set(1, 20);
+ vector.setNull(2);
+ vector.set(3, 30);
+ vector.set(4, 40);
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromInt32();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof IntBlock);
+ IntBlock intBlock = (IntBlock) block;
+
+ assertEquals(5, intBlock.getPositionCount());
+ assertEquals(10, intBlock.getInt(0));
+ assertEquals(20, intBlock.getInt(1));
+ assertTrue(intBlock.isNull(2));
+ assertEquals(30, intBlock.getInt(3));
+ assertEquals(40, intBlock.getInt(4));
+ }
+ }
+ }
+
+ public void testFromBoolean() {
+ try (BitVector vector = new BitVector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, 1);
+ vector.set(1, 0);
+ vector.setNull(2);
+ vector.set(3, 1);
+ vector.set(4, 0);
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromBoolean();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof BooleanBlock);
+ BooleanBlock booleanBlock = (BooleanBlock) block;
+
+ assertEquals(5, booleanBlock.getPositionCount());
+ assertTrue(booleanBlock.getBoolean(0));
+ assertFalse(booleanBlock.getBoolean(1));
+ assertTrue(booleanBlock.isNull(2));
+ assertTrue(booleanBlock.getBoolean(3));
+ assertFalse(booleanBlock.getBoolean(4));
+ }
+ }
+ }
+
+ public void testFromVarChar() {
+ try (VarCharVector vector = new VarCharVector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, "hello".getBytes(StandardCharsets.UTF_8));
+ vector.set(1, "world".getBytes(StandardCharsets.UTF_8));
+ vector.setNull(2);
+ vector.set(3, "foo".getBytes(StandardCharsets.UTF_8));
+ vector.set(4, "bar".getBytes(StandardCharsets.UTF_8));
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromVarChar();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof BytesRefBlock);
+ BytesRefBlock bytesRefBlock = (BytesRefBlock) block;
+
+ assertEquals(5, bytesRefBlock.getPositionCount());
+ assertEquals(new BytesRef("hello"), bytesRefBlock.getBytesRef(0, new BytesRef()));
+ assertEquals(new BytesRef("world"), bytesRefBlock.getBytesRef(1, new BytesRef()));
+ assertTrue(bytesRefBlock.isNull(2));
+ assertEquals(new BytesRef("foo"), bytesRefBlock.getBytesRef(3, new BytesRef()));
+ assertEquals(new BytesRef("bar"), bytesRefBlock.getBytesRef(4, new BytesRef()));
+ }
+ }
+ }
+
+ public void testFromVarBinary() {
+ try (VarBinaryVector vector = new VarBinaryVector("test", allocator)) {
+ vector.allocateNew(5);
+ vector.set(0, new byte[] { 1, 2, 3 });
+ vector.set(1, new byte[] { 4, 5, 6 });
+ vector.setNull(2);
+ vector.set(3, new byte[] { 7, 8, 9 });
+ vector.set(4, new byte[] { 10, 11, 12 });
+ vector.setValueCount(5);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromVarBinary();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof BytesRefBlock);
+ BytesRefBlock bytesRefBlock = (BytesRefBlock) block;
+
+ assertEquals(5, bytesRefBlock.getPositionCount());
+ assertEquals(new BytesRef(new byte[] { 1, 2, 3 }), bytesRefBlock.getBytesRef(0, new BytesRef()));
+ assertEquals(new BytesRef(new byte[] { 4, 5, 6 }), bytesRefBlock.getBytesRef(1, new BytesRef()));
+ assertTrue(bytesRefBlock.isNull(2));
+ assertEquals(new BytesRef(new byte[] { 7, 8, 9 }), bytesRefBlock.getBytesRef(3, new BytesRef()));
+ assertEquals(new BytesRef(new byte[] { 10, 11, 12 }), bytesRefBlock.getBytesRef(4, new BytesRef()));
+ }
+ }
+ }
+
+ public void testForTypeFactory() {
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.FLOAT8));
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.BIGINT));
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.INT));
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.BIT));
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.VARCHAR));
+ assertNotNull(ArrowToBlockConverter.forType(Types.MinorType.VARBINARY));
+ assertNull(ArrowToBlockConverter.forType(Types.MinorType.NULL));
+ assertNull(ArrowToBlockConverter.forType(Types.MinorType.STRUCT));
+ }
+
+ public void testFromFloat64EmptyVector() {
+ try (Float8Vector vector = new Float8Vector("test", allocator)) {
+ vector.allocateNew(0);
+ vector.setValueCount(0);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromFloat64();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof DoubleBlock);
+ DoubleBlock doubleBlock = (DoubleBlock) block;
+ assertEquals(0, doubleBlock.getPositionCount());
+ }
+ }
+ }
+
+ public void testFromInt32LargeVector() {
+ int size = 10000;
+ try (IntVector vector = new IntVector("test", allocator)) {
+ vector.allocateNew(size);
+ for (int i = 0; i < size; i++) {
+ if (i % 100 == 0) {
+ vector.setNull(i);
+ } else {
+ vector.set(i, i);
+ }
+ }
+ vector.setValueCount(size);
+
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromInt32();
+ try (Block block = converter.convert(vector, blockFactory)) {
+ assertTrue(block instanceof IntBlock);
+ IntBlock intBlock = (IntBlock) block;
+
+ assertEquals(size, intBlock.getPositionCount());
+ for (int i = 0; i < size; i++) {
+ if (i % 100 == 0) {
+ assertTrue("Position " + i + " should be null", intBlock.isNull(i));
+ } else {
+ assertEquals("Position " + i + " value mismatch", i, intBlock.getInt(i));
+ }
+ }
+ }
+ }
+ }
+
+ public void testSymmetricConversionDouble() {
+ // Test round-trip: Block → Arrow → Block
+ try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(3)) {
+ builder.appendDouble(1.5);
+ builder.appendNull();
+ builder.appendDouble(3.5);
+
+ try (DoubleBlock originalBlock = builder.build()) {
+ // Convert Block → Arrow using BlockConverter
+ try (Float8Vector vector = new Float8Vector("test", allocator)) {
+ vector.allocateNew(originalBlock.getPositionCount());
+ for (int i = 0; i < originalBlock.getPositionCount(); i++) {
+ if (originalBlock.isNull(i)) {
+ vector.setNull(i);
+ } else {
+ vector.set(i, originalBlock.getDouble(i));
+ }
+ }
+ vector.setValueCount(originalBlock.getPositionCount());
+
+ // Convert Arrow → Block using ArrowToBlockConverter
+ ArrowToBlockConverter converter = new ArrowToBlockConverter.FromFloat64();
+ try (Block convertedBlock = converter.convert(vector, blockFactory)) {
+ assertTrue(convertedBlock instanceof DoubleBlock);
+ DoubleBlock convertedDoubleBlock = (DoubleBlock) convertedBlock;
+
+ assertEquals(originalBlock.getPositionCount(), convertedDoubleBlock.getPositionCount());
+ for (int i = 0; i < originalBlock.getPositionCount(); i++) {
+ assertEquals(originalBlock.isNull(i), convertedDoubleBlock.isNull(i));
+ if (originalBlock.isNull(i) == false) {
+ assertEquals(originalBlock.getDouble(i), convertedDoubleBlock.getDouble(i), 0.0);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index c89138aa8207a..8166ceac5a0c5 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -16,6 +16,7 @@ import static org.elasticsearch.gradle.util.PlatformUtils.normalize
apply plugin: 'elasticsearch.internal-es-plugin'
apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-test-artifact'
apply plugin: 'elasticsearch.string-templates'
apply plugin: 'elasticsearch.publish'
@@ -48,7 +49,6 @@ dependencies {
api project(":libs:h3")
implementation project('arrow')
implementation "org.apache.commons:commons-math3:${versions.commons_math3}"
-
// Also contains a dummy processor to allow compilation with unused annotations.
annotationProcessor project('compute:gen')
@@ -96,6 +96,13 @@ tasks.named("dependencyLicenses").configure {
mapping from: /lucene-.*/, to: 'lucene'
}
+tasks.named("forbiddenPatterns").configure {
+ exclude '**/*.parquet'
+ exclude '**/*.avro'
+ exclude '**/.*.crc'
+}
+
+
def generatedPath = "src/main/generated"
def projectDirectory = project.layout.projectDirectory
def generatedSourceDir = projectDirectory.dir(generatedPath)
@@ -653,3 +660,4 @@ tasks.register("analyzePromqlQueries", JavaExec) {
classpath = sourceSets.test.runtimeClasspath
args project.findProperty("queriesFile") ?: "", project.findProperty("outputFile") ?: ""
}
+
diff --git a/x-pack/plugin/esql/qa/server/build.gradle b/x-pack/plugin/esql/qa/server/build.gradle
index 45d5adbf02ece..8e4e82c6ebcf3 100644
--- a/x-pack/plugin/esql/qa/server/build.gradle
+++ b/x-pack/plugin/esql/qa/server/build.gradle
@@ -8,4 +8,11 @@ dependencies {
// Requirement for some ESQL-specific utilities
implementation project(':x-pack:plugin:esql')
api project(xpackModule('esql:qa:testFixtures'))
+
+ // S3 fixture infrastructure for external source tests (Iceberg, Parquet)
+ api project(':test:fixtures:s3-fixture')
+ api project(':test:fixtures:aws-fixture-utils')
+
+ // Access to test utilities including IcebergS3FixtureUtils
+ api(project(path: xpackModule('esql'), configuration: 'testRuntimeElements'))
}
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
index 6571e1c7415b7..4c9094d509df5 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle
@@ -35,6 +35,9 @@ dependencies {
javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
javaRestTestImplementation project(xpackModule('esql:qa:server'))
javaRestTestImplementation project(xpackModule('esql'))
+
+ clusterPlugins project(xpackModule('esql-datasource-csv'))
+ clusterPlugins project(xpackModule('esql-datasource-http'))
}
GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest")
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle
index bd46073035979..a82642e9e1c99 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/build.gradle
@@ -23,6 +23,8 @@ dependencies {
javaRestTestImplementation project(xpackModule('esql'))
clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
+ clusterPlugins project(xpackModule('esql-datasource-csv'))
+ clusterPlugins project(xpackModule('esql-datasource-http'))
}
def supportedVersion = bwcVersion -> {
diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
index 9ae546ad23a58..712697e49b436 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
@@ -18,6 +18,8 @@ dependencies {
clusterPlugins project(':plugins:mapper-size')
clusterPlugins project(':plugins:mapper-murmur3')
clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
+ clusterPlugins project(xpackModule('esql-datasource-csv'))
+ clusterPlugins project(xpackModule('esql-datasource-http'))
}
GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest")
diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle
index 28954127d231f..be16a0a44d6c3 100644
--- a/x-pack/plugin/esql/qa/server/single-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle
@@ -32,6 +32,8 @@ dependencies {
clusterPlugins project(':plugins:mapper-size')
clusterPlugins project(':plugins:mapper-murmur3')
clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
+ clusterPlugins project(xpackModule('esql-datasource-csv'))
+ clusterPlugins project(xpackModule('esql-datasource-http'))
}
restResources {
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/datasources/S3FixtureUtils.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/datasources/S3FixtureUtils.java
new file mode 100644
index 0000000000000..411357ed307f2
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/datasources/S3FixtureUtils.java
@@ -0,0 +1,531 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.datasources;
+
+import fixture.s3.S3ConsistencyModel;
+import fixture.s3.S3HttpFixture;
+import fixture.s3.S3HttpHandler;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.BiPredicate;
+import java.util.stream.Collectors;
+
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+
+/**
+ * Shared utilities for S3 fixture-based integration tests.
+ * Provides common S3 fixture infrastructure for testing external data sources like Iceberg and Parquet.
+ */
+public final class S3FixtureUtils {
+
+ private static final Logger logger = LogManager.getLogger(S3FixtureUtils.class);
+
+ /** Default S3 access key for test fixtures */
+ public static final String ACCESS_KEY = "test-access-key";
+
+ /** Default S3 secret key for test fixtures */
+ public static final String SECRET_KEY = "test-secret-key";
+
+ /** Default bucket name for test fixtures */
+ public static final String BUCKET = "test-bucket";
+
+ /** Default warehouse path within the bucket */
+ public static final String WAREHOUSE = "warehouse";
+
+ /** Resource path for test fixtures */
+ private static final String FIXTURES_RESOURCE_PATH = "/iceberg-fixtures";
+
+ /** Thread-safe list of S3 request logs */
+ private static final CopyOnWriteArrayList requestLogs = new CopyOnWriteArrayList<>();
+
+ /** Set of known/expected S3 request types */
+ private static final Set KNOWN_REQUEST_TYPES = Set.of(
+ "GET_OBJECT",
+ "HEAD_OBJECT",
+ "PUT_OBJECT",
+ "DELETE_OBJECT",
+ "LIST_OBJECTS",
+ "LIST_OBJECTS_V2",
+ "INITIATE_MULTIPART",
+ "UPLOAD_PART",
+ "COMPLETE_MULTIPART",
+ "ABORT_MULTIPART",
+ "LIST_MULTIPART_UPLOADS",
+ "MULTI_OBJECT_DELETE"
+ );
+
+ /** Set of unsupported operations encountered during test execution */
+ private static final Set unsupportedOperations = ConcurrentHashMap.newKeySet();
+
+ private S3FixtureUtils() {
+ // Utility class - no instantiation
+ }
+
+ /**
+ * Get the warehouse path for S3 URLs.
+ */
+ public static String getWarehousePath() {
+ return WAREHOUSE;
+ }
+
+ /**
+ * Get all recorded S3 request logs.
+ */
+ public static List getRequestLogs() {
+ return Collections.unmodifiableList(new ArrayList<>(requestLogs));
+ }
+
+ /**
+ * Clear all recorded S3 request logs.
+ */
+ public static void clearRequestLogs() {
+ requestLogs.clear();
+ unsupportedOperations.clear();
+ }
+
+ /**
+ * Print a summary of S3 requests to the logger.
+ */
+ public static void printRequestSummary() {
+ List logs = getRequestLogs();
+ if (logs.isEmpty()) {
+ logger.info("No S3 requests recorded");
+ return;
+ }
+
+ Map byType = logs.stream().collect(Collectors.groupingBy(S3RequestLog::getRequestType, Collectors.counting()));
+
+ logger.info("S3 Request Summary ({} total requests):", logs.size());
+ byType.entrySet()
+ .stream()
+ .sorted(Map.Entry.comparingByValue().reversed())
+ .forEach(entry -> logger.info(" {}: {}", entry.getKey(), entry.getValue()));
+ }
+
+ /**
+ * Get the count of requests of a specific type.
+ */
+ public static int getRequestCount(String requestType) {
+ return (int) requestLogs.stream().filter(log -> requestType.equals(log.getRequestType())).count();
+ }
+
+ /**
+ * Get all requests of a specific type.
+ */
+ public static List getRequestsByType(String requestType) {
+ return requestLogs.stream().filter(log -> requestType.equals(log.getRequestType())).collect(Collectors.toList());
+ }
+
+ /**
+ * Check if any unknown/unsupported request types were encountered.
+ */
+ public static boolean hasUnknownRequests() {
+ return requestLogs.stream().anyMatch(log -> KNOWN_REQUEST_TYPES.contains(log.getRequestType()) == false);
+ }
+
+ /**
+ * Get all unknown/unsupported requests.
+ */
+ public static List getUnknownRequests() {
+ return requestLogs.stream().filter(log -> KNOWN_REQUEST_TYPES.contains(log.getRequestType()) == false).collect(Collectors.toList());
+ }
+
+ /**
+ * Build an error message for unsupported S3 operations, or null if none.
+ */
+ public static String buildUnsupportedOperationsError() {
+ if (unsupportedOperations.isEmpty()) {
+ return null;
+ }
+ return "Unsupported S3 operations encountered: " + String.join(", ", unsupportedOperations);
+ }
+
+ /**
+ * Add a blob to the S3 fixture.
+ */
+ public static void addBlobToFixture(S3HttpHandler handler, String key, String content) {
+ addBlobToFixture(handler, key, content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ /**
+ * Add a blob to the S3 fixture.
+ */
+ public static void addBlobToFixture(S3HttpHandler handler, String key, byte[] content) {
+ String fullPath = "/" + BUCKET + "/" + key;
+ handler.blobs().put(fullPath, new BytesArray(content));
+ logRequest("PUT_OBJECT", fullPath, content.length);
+ }
+
+ /**
+ * Log an S3 request.
+ */
+ private static void logRequest(String requestType, String path, long contentLength) {
+ requestLogs.add(new S3RequestLog(requestType, path, contentLength, System.currentTimeMillis()));
+ }
+
+ /**
+ * Create an S3FileIO configured to use the S3HttpFixture.
+ * This method uses reflection to avoid compile-time dependency on Iceberg.
+ * The Iceberg dependencies must be on the classpath at runtime.
+ *
+ * @param endpoint the S3 endpoint URL
+ * @return an S3FileIO instance configured for the fixture
+ * @throws RuntimeException if Iceberg is not on the classpath
+ */
+ @SuppressWarnings("unchecked")
+ public static T createS3FileIO(String endpoint) {
+ return createS3FileIO(endpoint, ACCESS_KEY, SECRET_KEY);
+ }
+
+ /**
+ * Create an S3FileIO with custom credentials.
+ * This method uses reflection to avoid compile-time dependency on Iceberg.
+ * The Iceberg dependencies must be on the classpath at runtime.
+ *
+ * @param endpoint the S3 endpoint URL
+ * @param accessKey the S3 access key
+ * @param secretKey the S3 secret key
+ * @return an S3FileIO instance configured with the given credentials
+ * @throws RuntimeException if Iceberg is not on the classpath
+ */
+ @SuppressWarnings("unchecked")
+ public static T createS3FileIO(String endpoint, String accessKey, String secretKey) {
+ try {
+ // Use reflection to create S3FileIO to avoid compile-time dependency on Iceberg
+ // This allows the qa/server module to compile without Iceberg while still
+ // providing this utility for modules that have Iceberg on the classpath
+
+ Class s3FileIOClass = Class.forName("org.apache.iceberg.aws.s3.S3FileIO");
+ Class s3ClientClass = Class.forName("software.amazon.awssdk.services.s3.S3Client");
+ Class s3ClientBuilderClass = Class.forName("software.amazon.awssdk.services.s3.S3ClientBuilder");
+ Class awsBasicCredentialsClass = Class.forName("software.amazon.awssdk.auth.credentials.AwsBasicCredentials");
+ Class staticCredentialsProviderClass = Class.forName("software.amazon.awssdk.auth.credentials.StaticCredentialsProvider");
+ Class regionClass = Class.forName("software.amazon.awssdk.regions.Region");
+ Class urlConnectionHttpClientClass = Class.forName("software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient");
+ Class profileFileClass = Class.forName("software.amazon.awssdk.profiles.ProfileFile");
+
+ // Create credentials
+ Object credentials = awsBasicCredentialsClass.getMethod("create", String.class, String.class)
+ .invoke(null, accessKey, secretKey);
+ Object credentialsProvider = staticCredentialsProviderClass.getMethod(
+ "create",
+ Class.forName("software.amazon.awssdk.auth.credentials.AwsCredentials")
+ ).invoke(null, credentials);
+
+ // Get US_EAST_1 region
+ Object usEast1Region = regionClass.getField("US_EAST_1").get(null);
+
+ // Create HTTP client
+ Object httpClientBuilder = urlConnectionHttpClientClass.getMethod("builder").invoke(null);
+ Object httpClient = httpClientBuilder.getClass().getMethod("build").invoke(httpClientBuilder);
+
+ // Create empty profile file
+ Object profileFileBuilder = profileFileClass.getMethod("builder").invoke(null);
+ Object credentialsType = Class.forName("software.amazon.awssdk.profiles.ProfileFile$Type").getField("CREDENTIALS").get(null);
+ profileFileBuilder.getClass()
+ .getMethod("type", Class.forName("software.amazon.awssdk.profiles.ProfileFile$Type"))
+ .invoke(profileFileBuilder, credentialsType);
+ profileFileBuilder.getClass()
+ .getMethod("content", InputStream.class)
+ .invoke(profileFileBuilder, new java.io.ByteArrayInputStream(new byte[0]));
+ Object emptyProfileFile = profileFileBuilder.getClass().getMethod("build").invoke(profileFileBuilder);
+
+ // Create S3Client using a supplier lambda
+ java.util.function.Supplier s3ClientSupplier = () -> {
+ try {
+ Object builder = s3ClientClass.getMethod("builder").invoke(null);
+
+ // Set credentials
+ builder.getClass()
+ .getMethod("credentialsProvider", Class.forName("software.amazon.awssdk.auth.credentials.AwsCredentialsProvider"))
+ .invoke(builder, credentialsProvider);
+
+ // Set endpoint if provided
+ if (endpoint != null) {
+ builder.getClass().getMethod("endpointOverride", java.net.URI.class).invoke(builder, java.net.URI.create(endpoint));
+ }
+
+ // Set region
+ builder.getClass().getMethod("region", regionClass).invoke(builder, usEast1Region);
+
+ // Enable path-style access
+ builder.getClass().getMethod("forcePathStyle", Boolean.class).invoke(builder, true);
+
+ // Set HTTP client
+ builder.getClass()
+ .getMethod("httpClient", Class.forName("software.amazon.awssdk.http.SdkHttpClient"))
+ .invoke(builder, httpClient);
+
+ return builder.getClass().getMethod("build").invoke(builder);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create S3Client", e);
+ }
+ };
+
+ // Create SerializableSupplier wrapper
+ Class serializableSupplierClass = Class.forName("org.apache.iceberg.util.SerializableSupplier");
+
+ // Create a dynamic proxy that implements SerializableSupplier
+ Object serializableSupplier = java.lang.reflect.Proxy.newProxyInstance(
+ Thread.currentThread().getContextClassLoader(),
+ new Class[] { serializableSupplierClass, java.io.Serializable.class },
+ (proxy, method, args) -> {
+ if ("get".equals(method.getName())) {
+ return s3ClientSupplier.get();
+ }
+ return method.invoke(s3ClientSupplier, args);
+ }
+ );
+
+ // Create S3FileIO with the supplier
+ return (T) s3FileIOClass.getConstructor(serializableSupplierClass).newInstance(serializableSupplier);
+
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(
+ "Iceberg or AWS SDK classes not found on classpath. " + "Ensure iceberg-aws and AWS SDK dependencies are available.",
+ e
+ );
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create S3FileIO via reflection", e);
+ }
+ }
+
+ /**
+ * Record of an S3 request for logging and analysis.
+ */
+ public static class S3RequestLog {
+ private final String requestType;
+ private final String path;
+ private final long contentLength;
+ private final long timestamp;
+
+ public S3RequestLog(String requestType, String path, long contentLength, long timestamp) {
+ this.requestType = requestType;
+ this.path = path;
+ this.contentLength = contentLength;
+ this.timestamp = timestamp;
+ }
+
+ public String getRequestType() {
+ return requestType;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public long getContentLength() {
+ return contentLength;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[%s] %s (%d bytes)", requestType, path, contentLength);
+ }
+ }
+
+ /**
+ * Extended S3HttpFixture that automatically loads test fixtures from resources.
+ * This fixture provides an in-memory S3-compatible endpoint for integration tests.
+ */
+ public static class DataSourcesS3HttpFixture extends S3HttpFixture {
+
+ private static final Logger fixtureLogger = LogManager.getLogger(DataSourcesS3HttpFixture.class);
+
+ private final int fixedPort;
+ private S3HttpHandler handler;
+
+ /**
+ * Create a fixture with a random available port.
+ */
+ public DataSourcesS3HttpFixture() {
+ this(-1);
+ }
+
+ /**
+ * Create a fixture with a specific port.
+ */
+ public DataSourcesS3HttpFixture(int port) {
+ super(true, () -> S3ConsistencyModel.STRONG_MPUS);
+ this.fixedPort = port;
+ }
+
+ @Override
+ protected S3HttpHandler createHandler() {
+ BiPredicate authPredicate = fixedAccessKey(ACCESS_KEY, () -> "us-east-1", "s3");
+ handler = new LoggingS3HttpHandler(BUCKET, WAREHOUSE, S3ConsistencyModel.STRONG_MPUS, authPredicate);
+ return handler;
+ }
+
+ /**
+ * Get the underlying S3HttpHandler for direct blob manipulation.
+ */
+ public S3HttpHandler getHandler() {
+ return handler;
+ }
+
+ /**
+ * Load test fixtures from the classpath resources into the S3 fixture.
+ */
+ public void loadFixturesFromResources() {
+ try {
+ URL resourceUrl = getClass().getResource(FIXTURES_RESOURCE_PATH);
+ if (resourceUrl == null) {
+ fixtureLogger.warn("Fixtures resource path not found: {}", FIXTURES_RESOURCE_PATH);
+ return;
+ }
+
+ if (resourceUrl.getProtocol().equals("file")) {
+ Path fixturesPath = Paths.get(resourceUrl.toURI());
+ loadFixturesFromPath(fixturesPath);
+ } else {
+ fixtureLogger.warn("Cannot load fixtures from non-file URL: {}", resourceUrl);
+ }
+ } catch (Exception e) {
+ fixtureLogger.error("Failed to load fixtures from resources", e);
+ }
+ }
+
+ private void loadFixturesFromPath(Path fixturesPath) throws IOException {
+ if (Files.exists(fixturesPath) == false) {
+ fixtureLogger.warn("Fixtures path does not exist: {}", fixturesPath);
+ return;
+ }
+
+ Set loadedFiles = new HashSet<>();
+
+ Files.walkFileTree(fixturesPath, new SimpleFileVisitor<>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ String relativePath = fixturesPath.relativize(file).toString();
+ String key = WAREHOUSE + "/" + relativePath;
+
+ byte[] content = Files.readAllBytes(file);
+ addBlobToFixture(handler, key, content);
+ loadedFiles.add(key);
+
+ return FileVisitResult.CONTINUE;
+ }
+ });
+
+ fixtureLogger.info("Loaded {} fixture files from {}", loadedFiles.size(), fixturesPath);
+ }
+
+ /**
+ * Load a single fixture file from an input stream.
+ */
+ public void loadFixture(String key, InputStream inputStream) throws IOException {
+ byte[] content = inputStream.readAllBytes();
+ addBlobToFixture(handler, key, content);
+ }
+ }
+
+ /**
+ * S3HttpHandler that logs all requests for analysis.
+ */
+ private static class LoggingS3HttpHandler extends S3HttpHandler {
+
+ private final BiPredicate authPredicate;
+
+ LoggingS3HttpHandler(
+ String bucket,
+ String basePath,
+ S3ConsistencyModel consistencyModel,
+ BiPredicate authPredicate
+ ) {
+ super(bucket, basePath, consistencyModel);
+ this.authPredicate = authPredicate;
+ }
+
+ @Override
+ public void handle(com.sun.net.httpserver.HttpExchange exchange) throws IOException {
+ String method = exchange.getRequestMethod();
+ String path = exchange.getRequestURI().getPath();
+ String query = exchange.getRequestURI().getQuery();
+
+ String requestType = classifyRequest(method, path, query);
+ logRequest(requestType, path, 0);
+
+ try {
+ // Allow unauthenticated access when no Authorization header is present.
+ // This enables plain HTTP clients (no S3 credentials) to read files from the fixture
+ // while still verifying S3 auth when credentials are sent (e.g. from the AWS SDK).
+ // NOTE: This means S3 auth bugs that cause missing Authorization headers will NOT
+ // be caught by this fixture -- only requests that send incorrect credentials are rejected.
+ String authHeader = exchange.getRequestHeaders().getFirst("Authorization");
+ if (authPredicate == null
+ || authHeader == null
+ || fixture.aws.AwsCredentialsUtils.checkAuthorization(authPredicate, exchange)) {
+ super.handle(exchange);
+ }
+ } catch (Exception e) {
+ logger.error("Error handling S3 request: {} {}", method, path, e);
+ throw e;
+ }
+ }
+
+ private String classifyRequest(String method, String path, String query) {
+ if ("GET".equals(method)) {
+ if (query != null && query.contains("list-type=2")) {
+ return "LIST_OBJECTS_V2";
+ } else if (query != null && query.contains("prefix=")) {
+ return "LIST_OBJECTS";
+ } else if (query != null && query.contains("uploads")) {
+ return "LIST_MULTIPART_UPLOADS";
+ }
+ return "GET_OBJECT";
+ } else if ("HEAD".equals(method)) {
+ return "HEAD_OBJECT";
+ } else if ("PUT".equals(method)) {
+ if (query != null && query.contains("uploadId=") && query.contains("partNumber=")) {
+ return "UPLOAD_PART";
+ }
+ return "PUT_OBJECT";
+ } else if ("DELETE".equals(method)) {
+ if (query != null && query.contains("uploadId=")) {
+ return "ABORT_MULTIPART";
+ }
+ return "DELETE_OBJECT";
+ } else if ("POST".equals(method)) {
+ if (query != null && query.contains("uploads")) {
+ return "INITIATE_MULTIPART";
+ } else if (query != null && query.contains("uploadId=")) {
+ return "COMPLETE_MULTIPART";
+ } else if (query != null && query.contains("delete")) {
+ return "MULTI_OBJECT_DELETE";
+ }
+ return "UNKNOWN_POST";
+ }
+ return "UNKNOWN_" + method;
+ }
+ }
+}
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AbstractExternalSourceSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AbstractExternalSourceSpecTestCase.java
new file mode 100644
index 0000000000000..b373cd791fc9a
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AbstractExternalSourceSpecTestCase.java
@@ -0,0 +1,424 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.qa.rest;
+
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.SpecReader;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.DataSourcesS3HttpFixture;
+import org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.S3RequestLog;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.ACCESS_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.BUCKET;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.SECRET_KEY;
+import static org.elasticsearch.xpack.esql.datasources.S3FixtureUtils.WAREHOUSE;
+
+/**
+ * Abstract base class for external source integration tests using S3HttpFixture.
+ * Provides common S3 fixture infrastructure for testing external data sources like Iceberg and Parquet.
+ *
+ * This class provides template-based query transformation where templates like {@code {{employees}}}
+ * are replaced with actual paths based on the storage backend (S3, HTTP, LOCAL) and format (parquet, csv).
+ *
+ * Subclasses specify the storage backend and format in their constructor, and the base class handles
+ * all path resolution automatically.
+ *
+ * @see S3FixtureUtils for shared S3 fixture utilities
+ */
+public abstract class AbstractExternalSourceSpecTestCase extends EsqlSpecTestCase {
+
+ private static final Logger logger = LogManager.getLogger(AbstractExternalSourceSpecTestCase.class);
+
+ /** Pattern to match template placeholders like {{employees}} */
+ private static final Pattern TEMPLATE_PATTERN = Pattern.compile("\\{\\{(\\w+)}}");
+
+ /** Base path for fixtures within the resource directory */
+ private static final String FIXTURES_BASE = "standalone";
+
+ /**
+ * Storage backend for accessing external files.
+ */
+ public enum StorageBackend {
+ /** S3 storage via S3HttpFixture */
+ S3,
+ /** HTTP storage via S3HttpFixture (same endpoint, different protocol) */
+ HTTP,
+ /** Local file system storage (direct classpath resource access) */
+ LOCAL
+ }
+
+ private static final List BACKENDS = List.of(StorageBackend.S3, StorageBackend.HTTP, StorageBackend.LOCAL);
+
+ /**
+ * Load csv-spec files matching the given patterns and cross-product each test with all storage backends.
+ * Returns parameter arrays suitable for a {@code @ParametersFactory} constructor with 7 arguments:
+ * (fileName, groupName, testName, lineNumber, testCase, instructions, storageBackend).
+ */
+ protected static List readExternalSpecTests(String... specPatterns) throws Exception {
+ List urls = new ArrayList<>();
+ for (String pattern : specPatterns) {
+ urls.addAll(classpathResources(pattern));
+ }
+ if (urls.isEmpty()) {
+ throw new IllegalStateException("No csv-spec files found for patterns: " + List.of(specPatterns));
+ }
+
+ List baseTests = SpecReader.readScriptSpec(urls, specParser());
+ List parameterizedTests = new ArrayList<>();
+ for (Object[] baseTest : baseTests) {
+ for (StorageBackend backend : BACKENDS) {
+ int baseLength = baseTest.length;
+ Object[] parameterizedTest = new Object[baseLength + 1];
+ System.arraycopy(baseTest, 0, parameterizedTest, 0, baseLength);
+ parameterizedTest[baseLength] = backend;
+ parameterizedTests.add(parameterizedTest);
+ }
+ }
+ return parameterizedTests;
+ }
+
+ @ClassRule
+ public static DataSourcesS3HttpFixture s3Fixture = new DataSourcesS3HttpFixture();
+
+ /** Cached path to local fixtures directory */
+ private static Path localFixturesPath;
+
+ /**
+ * Load fixtures from src/test/resources/iceberg-fixtures/ into the S3 fixture.
+ * This runs once before all tests, making pre-built test data available automatically.
+ */
+ @BeforeClass
+ public static void loadExternalSourceFixtures() {
+ s3Fixture.loadFixturesFromResources();
+ resolveLocalFixturesPath();
+ }
+
+ /**
+ * Resolve and cache the local path to the fixtures directory.
+ * This is used for LOCAL storage backend to access files directly from the classpath.
+ */
+ private static void resolveLocalFixturesPath() {
+ try {
+ URL resourceUrl = AbstractExternalSourceSpecTestCase.class.getResource("/iceberg-fixtures");
+ if (resourceUrl != null && resourceUrl.getProtocol().equals("file")) {
+ localFixturesPath = Paths.get(resourceUrl.toURI());
+ logger.info("Local fixtures path: {}", localFixturesPath);
+ } else {
+ logger.warn("Could not resolve local fixtures path - LOCAL storage backend may not work");
+ }
+ } catch (URISyntaxException e) {
+ logger.warn("Failed to resolve local fixtures path", e);
+ }
+ }
+
+ /**
+ * Skip standard test data loading for external source tests.
+ */
+ @BeforeClass
+ public static void skipStandardDataLoading() {
+ try {
+ java.lang.reflect.Field ingestField = EsqlSpecTestCase.class.getDeclaredField("INGEST");
+ ingestField.setAccessible(true);
+ Object ingest = ingestField.get(null);
+
+ java.lang.reflect.Field completedField = ingest.getClass().getDeclaredField("completed");
+ completedField.setAccessible(true);
+ completedField.setBoolean(ingest, true);
+
+ logger.info("Skipped standard test data loading for external source tests");
+ } catch (Exception e) {
+ logger.warn("Failed to skip standard data loading, tests may be slower", e);
+ }
+ }
+
+ @BeforeClass
+ public static void verifySetup() {
+ logger.info("=== External Source Test Setup Verification ===");
+ logger.info("S3 Fixture endpoint: {}", s3Fixture.getAddress());
+ logger.info("Local fixtures path: {}", localFixturesPath);
+ }
+
+ /**
+ * Automatically checks for unsupported S3 operations after each test.
+ */
+ @org.junit.After
+ public void checkForUnsupportedOperations() {
+ String errorMessage = S3FixtureUtils.buildUnsupportedOperationsError();
+ if (errorMessage != null) {
+ fail(errorMessage);
+ }
+ }
+
+ private final StorageBackend storageBackend;
+ private final String format;
+
+ protected AbstractExternalSourceSpecTestCase(
+ String fileName,
+ String groupName,
+ String testName,
+ Integer lineNumber,
+ CsvTestCase testCase,
+ String instructions,
+ StorageBackend storageBackend,
+ String format
+ ) {
+ super(fileName, groupName, testName, lineNumber, testCase, instructions);
+ this.storageBackend = storageBackend;
+ this.format = format;
+ }
+
+ /**
+ * Get the storage backend for this test.
+ */
+ protected StorageBackend getStorageBackend() {
+ return storageBackend;
+ }
+
+ /**
+ * Get the format (e.g., "parquet", "csv") for this test.
+ */
+ protected String getFormat() {
+ return format;
+ }
+
+ @Override
+ protected void shouldSkipTest(String testName) throws IOException {
+ // skip nothing
+ // super skips tests for the "regular" CsvTest/EsqlSpecIT suites
+ }
+
+ /**
+ * Override doTest() to transform templates and inject storage-specific parameters.
+ */
+ @Override
+ protected void doTest() throws Throwable {
+ String query = testCase.query;
+
+ if (query.contains(MULTIFILE_SUFFIX)) {
+ // HTTP does not support directory listing, so skip multi-file glob tests
+ assumeTrue("HTTP backend does not support multi-file glob patterns", storageBackend != StorageBackend.HTTP);
+ // CSV format does not yet support multi-file glob patterns
+ assumeTrue("CSV format does not support multi-file glob patterns", "csv".equals(format) == false);
+
+ }
+
+ // Transform templates like {{employees}} to actual paths
+ query = transformTemplates(query);
+
+ // Inject endpoint and credentials for S3 backend
+ if (storageBackend == StorageBackend.S3 && isExternalQuery(query) && hasEndpointParam(query) == false) {
+ query = injectS3Params(query);
+ }
+
+ logger.debug("Transformed query for {} backend: {}", storageBackend, query);
+ doTest(query);
+ }
+
+ /**
+ * Transform template placeholders in the query.
+ * Replaces {{anything}} with the actual path based on storage backend and format.
+ *
+ * @param query the query with template placeholders
+ * @return the query with templates replaced by actual paths
+ */
+ private String transformTemplates(String query) {
+ Matcher matcher = TEMPLATE_PATTERN.matcher(query);
+ StringBuffer result = new StringBuffer();
+
+ while (matcher.find()) {
+ String templateName = matcher.group(1);
+ String resolvedPath = resolveTemplatePath(templateName);
+ matcher.appendReplacement(result, Matcher.quoteReplacement(resolvedPath));
+ }
+ matcher.appendTail(result);
+
+ return result.toString();
+ }
+
+ /** Suffix that triggers multi-file glob resolution */
+ private static final String MULTIFILE_SUFFIX = "_multifile";
+
+ /**
+ * Resolve a template name to an actual path based on storage backend and format.
+ *
+ * @param templateName the template name (e.g., "employees" or "employees_multifile")
+ * @return the resolved path
+ */
+ private String resolveTemplatePath(String templateName) {
+ String relativePath;
+ if (templateName.endsWith(MULTIFILE_SUFFIX)) {
+ // Multi-file template: employees_multifile -> multifile/*.parquet
+ relativePath = "multifile/*." + format;
+ } else {
+ // Single-file template: employees -> standalone/employees.parquet
+ String filename = templateName + "." + format;
+ relativePath = FIXTURES_BASE + "/" + filename;
+ }
+
+ switch (storageBackend) {
+ case S3:
+ // S3 path: s3://bucket/warehouse/standalone/employees.parquet
+ return "s3://" + BUCKET + "/" + WAREHOUSE + "/" + relativePath;
+
+ case HTTP:
+ // HTTP path: http://host:port/bucket/warehouse/standalone/employees.parquet
+ return s3Fixture.getAddress() + "/" + BUCKET + "/" + WAREHOUSE + "/" + relativePath;
+
+ case LOCAL:
+ // Local path: file:///absolute/path/to/iceberg-fixtures/standalone/employees.parquet
+ if (localFixturesPath != null) {
+ Path localFile = localFixturesPath.resolve(relativePath);
+ return "file://" + localFile.toAbsolutePath().toString();
+ } else {
+ // Fallback to S3 if local path not available
+ logger.warn("Local fixtures path not available, falling back to S3");
+ return "s3://" + BUCKET + "/" + WAREHOUSE + "/" + relativePath;
+ }
+
+ default:
+ throw new IllegalArgumentException("Unknown storage backend: " + storageBackend);
+ }
+ }
+
+ /**
+ * Inject S3 endpoint and credentials into the query.
+ */
+ private String injectS3Params(String query) {
+ String trimmed = query.trim();
+ int pipeIndex = findFirstPipeAfterExternal(trimmed);
+
+ String externalPart;
+ String restOfQuery;
+
+ if (pipeIndex == -1) {
+ externalPart = trimmed;
+ restOfQuery = "";
+ } else {
+ externalPart = trimmed.substring(0, pipeIndex).trim();
+ restOfQuery = " " + trimmed.substring(pipeIndex);
+ }
+
+ StringBuilder params = new StringBuilder();
+ params.append(" WITH { ");
+ params.append("\"endpoint\": \"").append(s3Fixture.getAddress()).append("\", ");
+ params.append("\"access_key\": \"").append(ACCESS_KEY).append("\", ");
+ params.append("\"secret_key\": \"").append(SECRET_KEY).append("\"");
+ params.append(" }");
+
+ return externalPart + params.toString() + restOfQuery;
+ }
+
+ /**
+ * Check if query starts with EXTERNAL command.
+ */
+ private static boolean isExternalQuery(String query) {
+ return query.trim().toUpperCase(Locale.ROOT).startsWith("EXTERNAL");
+ }
+
+ /**
+ * Check if query already has endpoint parameter.
+ */
+ private static boolean hasEndpointParam(String query) {
+ return query.toLowerCase(Locale.ROOT).contains("endpoint");
+ }
+
+ /**
+ * Find the first pipe character that's not inside a quoted string.
+ */
+ private static int findFirstPipeAfterExternal(String query) {
+ boolean inQuotes = false;
+ char quoteChar = 0;
+
+ for (int i = 0; i < query.length(); i++) {
+ char c = query.charAt(i);
+
+ if (inQuotes == false && (c == '"' || c == '\'')) {
+ inQuotes = true;
+ quoteChar = c;
+ } else if (inQuotes && c == quoteChar) {
+ inQuotes = false;
+ } else if (inQuotes == false && c == '|') {
+ return i;
+ }
+ }
+
+ return -1;
+ }
+
+ @Override
+ protected boolean supportsInferenceTestServiceOnLocalCluster() {
+ return false;
+ }
+
+ @Override
+ protected boolean supportsSemanticTextInference() {
+ return false;
+ }
+
+ // Static utility methods for fixture access
+
+ protected static String getS3Endpoint() {
+ return s3Fixture.getAddress();
+ }
+
+ protected static List getRequestLogs() {
+ return S3FixtureUtils.getRequestLogs();
+ }
+
+ protected static void clearRequestLogs() {
+ S3FixtureUtils.clearRequestLogs();
+ }
+
+ protected static void printRequestSummary() {
+ S3FixtureUtils.printRequestSummary();
+ }
+
+ protected static int getRequestCount(String requestType) {
+ return S3FixtureUtils.getRequestCount(requestType);
+ }
+
+ protected static List getRequestsByType(String requestType) {
+ return S3FixtureUtils.getRequestsByType(requestType);
+ }
+
+ protected static boolean hasUnknownRequests() {
+ return S3FixtureUtils.hasUnknownRequests();
+ }
+
+ protected static List getUnknownRequests() {
+ return S3FixtureUtils.getUnknownRequests();
+ }
+
+ protected static void addBlobToFixture(String key, String content) {
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), key, content);
+ }
+
+ protected static void addBlobToFixture(String key, byte[] content) {
+ S3FixtureUtils.addBlobToFixture(s3Fixture.getHandler(), key, content);
+ }
+
+ protected static String getWarehousePath() {
+ return S3FixtureUtils.getWarehousePath();
+ }
+}
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
index 974eb9748e310..a2b8d2ca338d6 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
@@ -297,6 +297,12 @@ protected void shouldSkipTest(String testName) throws IOException {
if (supportsSourceFieldMapping() == false) {
assumeFalse("source mapping tests are muted", testCase.requiredCapabilities.contains(SOURCE_FIELD_MAPPING.capabilityName()));
}
+ // EXTERNAL command tests require dedicated infrastructure (S3 fixture, datasource plugins, template replacement)
+ // that is only available in AbstractExternalSourceSpecTestCase subclasses, not in generic EsqlSpecIT suites.
+ assumeFalse(
+ "EXTERNAL command tests require dedicated external source test infrastructure",
+ testCase.query.trim().toUpperCase(Locale.ROOT).startsWith("EXTERNAL")
+ );
}
protected static void checkCapabilities(
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-basic.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-basic.csv-spec
new file mode 100644
index 0000000000000..a040fc8750df6
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-basic.csv-spec
@@ -0,0 +1,198 @@
+// Shared tests for standalone external files (Parquet, CSV, etc.)
+// Uses {{employees}} template that gets replaced with the actual path based on storage backend and format
+
+readAllEmployees
+EXTERNAL "{{employees}}"
+| KEEP emp_no, first_name, last_name, birth_date, gender, hire_date, languages, height, salary, still_hired
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | last_name:keyword | birth_date:date | gender:keyword | hire_date:date | languages:integer | height:double | salary:integer | still_hired:boolean
+10001 | "Georgi" | "Facello" | 1953-09-02T00:00:00.000Z | "M" | 1986-06-26T00:00:00.000Z | 2 | 2.03 | 57305 | true
+10002 | "Bezalel" | "Simmel" | 1964-06-02T00:00:00.000Z | "F" | 1985-11-21T00:00:00.000Z | 5 | 2.08 | 56371 | true
+10003 | "Parto" | "Bamford" | 1959-12-03T00:00:00.000Z | "M" | 1986-08-28T00:00:00.000Z | 4 | 1.83 | 61805 | false
+10004 | "Chirstian" | "Koblick" | 1954-05-01T00:00:00.000Z | "M" | 1986-12-01T00:00:00.000Z | 5 | 1.78 | 36174 | true
+10005 | "Kyoichi" | "Maliniak" | 1955-01-21T00:00:00.000Z | "M" | 1989-09-12T00:00:00.000Z | 1 | 2.05 | 63528 | true
+;
+
+selectSpecificColumns
+EXTERNAL "{{employees}}"
+| KEEP emp_no, first_name, last_name, salary
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | last_name:keyword | salary:integer
+10001 | "Georgi" | "Facello" | 57305
+10002 | "Bezalel" | "Simmel" | 56371
+10003 | "Parto" | "Bamford" | 61805
+10004 | "Chirstian" | "Koblick" | 36174
+10005 | "Kyoichi" | "Maliniak" | 63528
+;
+
+filterByEmployeeNumber
+EXTERNAL "{{employees}}"
+| WHERE emp_no == 10001
+| KEEP emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10001 | "Georgi" | "Facello"
+;
+
+filterBySalaryRange
+EXTERNAL "{{employees}}"
+| WHERE salary > 60000 AND salary < 70000
+| KEEP emp_no, first_name, salary
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | salary:integer
+10003 | "Parto" | 61805
+10005 | "Kyoichi" | 63528
+10006 | "Anneke" | 60335
+10009 | "Sumant" | 66174
+10016 | "Kazuhito" | 61358
+;
+
+filterByGender
+EXTERNAL "{{employees}}"
+| WHERE gender == "F"
+| KEEP emp_no, first_name, last_name, gender
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | last_name:keyword | gender:keyword
+10002 | "Bezalel" | "Simmel" | "F"
+10006 | "Anneke" | "Preusig" | "F"
+10007 | "Tzvetan" | "Zielinski" | "F"
+;
+
+filterByEmploymentStatus
+EXTERNAL "{{employees}}"
+| WHERE still_hired == false
+| KEEP emp_no, first_name, last_name, still_hired
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | last_name:keyword | still_hired:boolean
+10003 | "Parto" | "Bamford" | false
+10006 | "Anneke" | "Preusig" | false
+10009 | "Sumant" | "Peac" | false
+;
+
+aggregateCount
+EXTERNAL "{{employees}}"
+| STATS count = COUNT(*);
+
+count:long
+100
+;
+
+aggregateByGender
+EXTERNAL "{{employees}}"
+| STATS count = COUNT(*) BY gender
+| SORT gender;
+
+count:long | gender:keyword
+33 | "F"
+57 | "M"
+10 | null
+;
+
+aggregateAverageSalary
+EXTERNAL "{{employees}}"
+| STATS avg_salary = AVG(salary);
+
+avg_salary:double
+48248.55
+;
+
+aggregateSalaryStats
+EXTERNAL "{{employees}}"
+| STATS min_salary = MIN(salary), max_salary = MAX(salary), avg_salary = AVG(salary);
+
+min_salary:integer | max_salary:integer | avg_salary:double
+25324 | 74999 | 48248.55
+;
+
+aggregateSalaryByGender
+EXTERNAL "{{employees}}"
+| STATS avg_salary = AVG(salary), count = COUNT(*) BY gender
+| SORT gender;
+
+avg_salary:double | count:long | gender:keyword
+50490.78787878788 | 33 | "F"
+46860.59649122807 | 57 | "M"
+48760.5 | 10 | null
+;
+
+filterAndSort
+EXTERNAL "{{employees}}"
+| WHERE salary > 70000
+| KEEP emp_no, first_name, salary
+| SORT salary DESC
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | salary:integer
+10029 | "Otmar" | 74999
+10045 | "Moss" | 74970
+10007 | "Tzvetan" | 74572
+10027 | "Divier" | 73851
+10019 | "Lillian" | 73717
+;
+
+evalComputedColumn
+EXTERNAL "{{employees}}"
+| EVAL annual_bonus = salary * 0.1
+| KEEP emp_no, first_name, salary, annual_bonus
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | salary:integer | annual_bonus:double
+10001 | "Georgi" | 57305 | 5730.5
+10002 | "Bezalel" | 56371 | 5637.1
+10003 | "Parto" | 61805 | 6180.5
+;
+
+complexQuery
+EXTERNAL "{{employees}}"
+| WHERE still_hired == true AND salary > 55000
+| EVAL salary_category = CASE(salary < 60000, "standard", salary < 70000, "senior", "principal")
+| STATS count = COUNT(*), avg_salary = AVG(salary) BY salary_category
+| SORT salary_category;
+
+count:long | avg_salary:double | salary_category:keyword
+2 | 74075.0 | "principal"
+5 | 67017.0 | "senior"
+4 | 56789.25 | "standard"
+;
+
+// Sub-field columns (languages.long, height.float, height.scaled_float, height.half_float)
+
+selectAdditionalColumns
+EXTERNAL "{{employees}}"
+| KEEP emp_no, first_name, `languages.long`, avg_worked_seconds
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | languages.long:long | avg_worked_seconds:long
+10001 | "Georgi" | 2 | 268728049
+10002 | "Bezalel" | 5 | 328922887
+10003 | "Parto" | 4 | 200296405
+10004 | "Chirstian" | 5 | 311267831
+10005 | "Kyoichi" | 1 | 244294991
+;
+
+selectHeightVariants
+EXTERNAL "{{employees}}"
+| EVAL height_float_rounded = ROUND(`height.float`, 2), height_half_float_rounded = ROUND(`height.half_float`, 2)
+| KEEP emp_no, height, height_float_rounded, `height.scaled_float`, height_half_float_rounded
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | height:double | height_float_rounded:double | height.scaled_float:double | height_half_float_rounded:double
+10001 | 2.03 | 2.03 | 2.03 | 2.03
+10002 | 2.08 | 2.08 | 2.08 | 2.08
+10003 | 1.83 | 1.83 | 1.83 | 1.83
+10004 | 1.78 | 1.78 | 1.78 | 1.78
+10005 | 2.05 | 2.05 | 2.05 | 2.05
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-multifile.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-multifile.csv-spec
new file mode 100644
index 0000000000000..95e0ad94462c7
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/external-multifile.csv-spec
@@ -0,0 +1,31 @@
+// Tests for reading data merged from multiple files via glob patterns.
+// Uses {{employees_multifile}} template which resolves to multifile/*.parquet (or *.csv).
+// Discovery correctness is validated in GlobDiscoveryLocalTests; these tests verify data merging.
+
+// AwaitsFix: multifile CSV test data (iceberg-fixtures/multifile/) not yet created; glob matches no files
+readAllEmployeesMultiFile-Ignore
+EXTERNAL "{{employees_multifile}}"
+| STATS count = COUNT(*);
+
+count:long
+100
+;
+
+aggregateMultiFileByGender-Ignore
+EXTERNAL "{{employees_multifile}}"
+| STATS count = COUNT(*) BY gender
+| SORT gender;
+
+count:long | gender:keyword
+33 | "F"
+57 | "M"
+10 | null
+;
+
+multiFileSalaryStats-Ignore
+EXTERNAL "{{employees_multifile}}"
+| STATS min_salary = MIN(salary), max_salary = MAX(salary), avg_salary = AVG(salary);
+
+min_salary:integer | max_salary:integer | avg_salary:double
+25324 | 74999 | 48248.55
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/iceberg-basic.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/iceberg-basic.csv-spec
new file mode 100644
index 0000000000000..9f74d78e0fc72
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/iceberg-basic.csv-spec
@@ -0,0 +1,206 @@
+// Tests for Iceberg tables with metadata
+
+simpleRow
+ROW a = 1, b = "iceberg";
+
+a:integer | b:keyword
+1 | "iceberg"
+;
+
+// Employees dataset: 100 rows, 23 columns (integers, keywords, dates, doubles, booleans, multi-values)
+
+readAllEmployees
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| KEEP emp_no, first_name, last_name, birth_date, gender, hire_date, languages, height, salary, still_hired
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | last_name:keyword | birth_date:date | gender:keyword | hire_date:date | languages:integer | height:double | salary:integer | still_hired:boolean
+10001 | "Georgi" | "Facello" | 1953-09-02T00:00:00.000Z | "M" | 1986-06-26T00:00:00.000Z | 2 | 2.03 | 57305 | true
+10002 | "Bezalel" | "Simmel" | 1964-06-02T00:00:00.000Z | "F" | 1985-11-21T00:00:00.000Z | 5 | 2.08 | 56371 | true
+10003 | "Parto" | "Bamford" | 1959-12-03T00:00:00.000Z | "M" | 1986-08-28T00:00:00.000Z | 4 | 1.83 | 61805 | false
+10004 | "Chirstian" | "Koblick" | 1954-05-01T00:00:00.000Z | "M" | 1986-12-01T00:00:00.000Z | 5 | 1.78 | 36174 | true
+10005 | "Kyoichi" | "Maliniak" | 1955-01-21T00:00:00.000Z | "M" | 1989-09-12T00:00:00.000Z | 1 | 2.05 | 63528 | true
+;
+
+selectSpecificColumns
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| KEEP emp_no, first_name, last_name, salary
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | last_name:keyword | salary:integer
+10001 | "Georgi" | "Facello" | 57305
+10002 | "Bezalel" | "Simmel" | 56371
+10003 | "Parto" | "Bamford" | 61805
+10004 | "Chirstian" | "Koblick" | 36174
+10005 | "Kyoichi" | "Maliniak" | 63528
+;
+
+filterByEmployeeNumber
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE emp_no == 10001
+| KEEP emp_no, first_name, last_name;
+
+emp_no:integer | first_name:keyword | last_name:keyword
+10001 | "Georgi" | "Facello"
+;
+
+filterBySalaryRange
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE salary > 60000 AND salary < 70000
+| KEEP emp_no, first_name, salary
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | salary:integer
+10003 | "Parto" | 61805
+10005 | "Kyoichi" | 63528
+10006 | "Anneke" | 60335
+10009 | "Sumant" | 66174
+10016 | "Kazuhito" | 61358
+;
+
+filterByGender
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE gender == "F"
+| KEEP emp_no, first_name, last_name, gender
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | last_name:keyword | gender:keyword
+10002 | "Bezalel" | "Simmel" | "F"
+10006 | "Anneke" | "Preusig" | "F"
+10007 | "Tzvetan" | "Zielinski" | "F"
+;
+
+filterByEmploymentStatus
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE still_hired == false
+| KEEP emp_no, first_name, last_name, still_hired
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | last_name:keyword | still_hired:boolean
+10003 | "Parto" | "Bamford" | false
+10006 | "Anneke" | "Preusig" | false
+10009 | "Sumant" | "Peac" | false
+;
+
+aggregateCount
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| STATS count = COUNT(*);
+
+count:long
+100
+;
+
+aggregateByGender
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| STATS count = COUNT(*) BY gender
+| SORT gender;
+
+count:long | gender:keyword
+33 | "F"
+57 | "M"
+10 | null
+;
+
+aggregateAverageSalary
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| STATS avg_salary = AVG(salary);
+
+avg_salary:double
+48248.55
+;
+
+aggregateSalaryStats
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| STATS min_salary = MIN(salary), max_salary = MAX(salary), avg_salary = AVG(salary);
+
+min_salary:integer | max_salary:integer | avg_salary:double
+25324 | 74999 | 48248.55
+;
+
+aggregateSalaryByGender
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| STATS avg_salary = AVG(salary), count = COUNT(*) BY gender
+| SORT gender;
+
+avg_salary:double | count:long | gender:keyword
+50490.78787878788 | 33 | "F"
+46860.59649122807 | 57 | "M"
+48760.5 | 10 | null
+;
+
+filterAndSort
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE salary > 70000
+| KEEP emp_no, first_name, salary
+| SORT salary DESC
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | salary:integer
+10029 | "Otmar" | 74999
+10045 | "Moss" | 74970
+10007 | "Tzvetan" | 74572
+10027 | "Divier" | 73851
+10019 | "Lillian" | 73717
+;
+
+evalComputedColumn
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| EVAL annual_bonus = salary * 0.1
+| KEEP emp_no, first_name, salary, annual_bonus
+| SORT emp_no
+| LIMIT 3;
+
+emp_no:integer | first_name:keyword | salary:integer | annual_bonus:double
+10001 | "Georgi" | 57305 | 5730.5
+10002 | "Bezalel" | 56371 | 5637.1
+10003 | "Parto" | 61805 | 6180.5
+;
+
+complexQuery
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| WHERE still_hired == true AND salary > 55000
+| EVAL salary_category = CASE(salary < 60000, "standard", salary < 70000, "senior", "principal")
+| STATS count = COUNT(*), avg_salary = AVG(salary) BY salary_category
+| SORT salary_category;
+
+count:long | avg_salary:double | salary_category:keyword
+2 | 74075.0 | "principal"
+5 | 67017.0 | "senior"
+4 | 56789.25 | "standard"
+;
+
+// Additional column types
+
+selectAdditionalColumns
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| KEEP emp_no, first_name, `languages.long`, avg_worked_seconds
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | first_name:keyword | languages.long:long | avg_worked_seconds:long
+10001 | "Georgi" | 2 | 268728049
+10002 | "Bezalel" | 5 | 328922887
+10003 | "Parto" | 4 | 200296405
+10004 | "Chirstian" | 5 | 311267831
+10005 | "Kyoichi" | 1 | 244294991
+;
+
+selectHeightVariants
+EXTERNAL "s3://iceberg-test/warehouse/employees"
+| EVAL height_float_rounded = ROUND(`height.float`, 2), height_half_float_rounded = ROUND(`height.half_float`, 2)
+| KEEP emp_no, height, height_float_rounded, `height.scaled_float`, height_half_float_rounded
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | height:double | height_float_rounded:double | height.scaled_float:double | height_half_float_rounded:double
+10001 | 2.03 | 2.03 | 2.03 | 2.03
+10002 | 2.08 | 2.08 | 2.08 | 2.08
+10003 | 1.83 | 1.83 | 1.83 | 1.83
+10004 | 1.78 | 1.78 | 1.78 | 1.78
+10005 | 2.05 | 2.05 | 2.05 | 2.05
+;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
index d7837af8eea10..2bb1a5499bd79 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
@@ -17,150 +17,151 @@ STATS=16
WHERE=17
FROM=18
TS=19
-FORK=20
-FUSE=21
-INLINE=22
-INLINESTATS=23
-JOIN_LOOKUP=24
-DEV_JOIN_FULL=25
-DEV_JOIN_LEFT=26
-DEV_JOIN_RIGHT=27
-DEV_LOOKUP=28
-DEV_MMR=29
-MV_EXPAND=30
-DROP=31
-KEEP=32
-DEV_INSIST=33
-PROMQL=34
-RENAME=35
-SET=36
-SHOW=37
-UNKNOWN_CMD=38
-CHANGE_POINT_LINE_COMMENT=39
-CHANGE_POINT_MULTILINE_COMMENT=40
-CHANGE_POINT_WS=41
-ENRICH_POLICY_NAME=42
-ENRICH_LINE_COMMENT=43
-ENRICH_MULTILINE_COMMENT=44
-ENRICH_WS=45
-ENRICH_FIELD_LINE_COMMENT=46
-ENRICH_FIELD_MULTILINE_COMMENT=47
-ENRICH_FIELD_WS=48
-EXPLAIN_WS=49
-EXPLAIN_LINE_COMMENT=50
-EXPLAIN_MULTILINE_COMMENT=51
-PIPE=52
-QUOTED_STRING=53
-INTEGER_LITERAL=54
-DECIMAL_LITERAL=55
-AND=56
-ASC=57
-ASSIGN=58
-BY=59
-CAST_OP=60
-COLON=61
-SEMICOLON=62
-COMMA=63
-DESC=64
-DOT=65
-FALSE=66
-FIRST=67
-IN=68
-IS=69
-LAST=70
-LIKE=71
-NOT=72
-NULL=73
-NULLS=74
-ON=75
-OR=76
-PARAM=77
-RLIKE=78
-TRUE=79
-WITH=80
-EQ=81
-CIEQ=82
-NEQ=83
-LT=84
-LTE=85
-GT=86
-GTE=87
-PLUS=88
-MINUS=89
-ASTERISK=90
-SLASH=91
-PERCENT=92
-LEFT_BRACES=93
-RIGHT_BRACES=94
-DOUBLE_PARAMS=95
-NAMED_OR_POSITIONAL_PARAM=96
-NAMED_OR_POSITIONAL_DOUBLE_PARAMS=97
-OPENING_BRACKET=98
-CLOSING_BRACKET=99
-LP=100
-RP=101
-UNQUOTED_IDENTIFIER=102
-QUOTED_IDENTIFIER=103
-EXPR_LINE_COMMENT=104
-EXPR_MULTILINE_COMMENT=105
-EXPR_WS=106
-METADATA=107
-UNQUOTED_SOURCE=108
-FROM_LINE_COMMENT=109
-FROM_MULTILINE_COMMENT=110
-FROM_WS=111
-FORK_WS=112
-FORK_LINE_COMMENT=113
-FORK_MULTILINE_COMMENT=114
-GROUP=115
-SCORE=116
-KEY=117
-FUSE_LINE_COMMENT=118
-FUSE_MULTILINE_COMMENT=119
-FUSE_WS=120
-INLINE_STATS=121
-INLINE_LINE_COMMENT=122
-INLINE_MULTILINE_COMMENT=123
-INLINE_WS=124
-JOIN=125
-USING=126
-JOIN_LINE_COMMENT=127
-JOIN_MULTILINE_COMMENT=128
-JOIN_WS=129
-LOOKUP_LINE_COMMENT=130
-LOOKUP_MULTILINE_COMMENT=131
-LOOKUP_WS=132
-LOOKUP_FIELD_LINE_COMMENT=133
-LOOKUP_FIELD_MULTILINE_COMMENT=134
-LOOKUP_FIELD_WS=135
-MMR_LIMIT=136
-MMR_LINE_COMMENT=137
-MMR_MULTILINE_COMMENT=138
-MMR_WS=139
-MVEXPAND_LINE_COMMENT=140
-MVEXPAND_MULTILINE_COMMENT=141
-MVEXPAND_WS=142
-ID_PATTERN=143
-PROJECT_LINE_COMMENT=144
-PROJECT_MULTILINE_COMMENT=145
-PROJECT_WS=146
-PROMQL_PARAMS_LINE_COMMENT=147
-PROMQL_PARAMS_MULTILINE_COMMENT=148
-PROMQL_PARAMS_WS=149
-PROMQL_QUERY_COMMENT=150
-PROMQL_SINGLE_QUOTED_STRING=151
-PROMQL_OTHER_QUERY_CONTENT=152
-AS=153
-RENAME_LINE_COMMENT=154
-RENAME_MULTILINE_COMMENT=155
-RENAME_WS=156
-SET_LINE_COMMENT=157
-SET_MULTILINE_COMMENT=158
-SET_WS=159
-INFO=160
-SHOW_LINE_COMMENT=161
-SHOW_MULTILINE_COMMENT=162
-SHOW_WS=163
+EXTERNAL=20
+FORK=21
+FUSE=22
+INLINE=23
+INLINESTATS=24
+JOIN_LOOKUP=25
+DEV_JOIN_FULL=26
+DEV_JOIN_LEFT=27
+DEV_JOIN_RIGHT=28
+DEV_LOOKUP=29
+DEV_MMR=30
+MV_EXPAND=31
+DROP=32
+KEEP=33
+DEV_INSIST=34
+PROMQL=35
+RENAME=36
+SET=37
+SHOW=38
+UNKNOWN_CMD=39
+CHANGE_POINT_LINE_COMMENT=40
+CHANGE_POINT_MULTILINE_COMMENT=41
+CHANGE_POINT_WS=42
+ENRICH_POLICY_NAME=43
+ENRICH_LINE_COMMENT=44
+ENRICH_MULTILINE_COMMENT=45
+ENRICH_WS=46
+ENRICH_FIELD_LINE_COMMENT=47
+ENRICH_FIELD_MULTILINE_COMMENT=48
+ENRICH_FIELD_WS=49
+EXPLAIN_WS=50
+EXPLAIN_LINE_COMMENT=51
+EXPLAIN_MULTILINE_COMMENT=52
+PIPE=53
+QUOTED_STRING=54
+INTEGER_LITERAL=55
+DECIMAL_LITERAL=56
+AND=57
+ASC=58
+ASSIGN=59
+BY=60
+CAST_OP=61
+COLON=62
+SEMICOLON=63
+COMMA=64
+DESC=65
+DOT=66
+FALSE=67
+FIRST=68
+IN=69
+IS=70
+LAST=71
+LIKE=72
+NOT=73
+NULL=74
+NULLS=75
+ON=76
+OR=77
+PARAM=78
+RLIKE=79
+TRUE=80
+WITH=81
+EQ=82
+CIEQ=83
+NEQ=84
+LT=85
+LTE=86
+GT=87
+GTE=88
+PLUS=89
+MINUS=90
+ASTERISK=91
+SLASH=92
+PERCENT=93
+LEFT_BRACES=94
+RIGHT_BRACES=95
+DOUBLE_PARAMS=96
+NAMED_OR_POSITIONAL_PARAM=97
+NAMED_OR_POSITIONAL_DOUBLE_PARAMS=98
+OPENING_BRACKET=99
+CLOSING_BRACKET=100
+LP=101
+RP=102
+UNQUOTED_IDENTIFIER=103
+QUOTED_IDENTIFIER=104
+EXPR_LINE_COMMENT=105
+EXPR_MULTILINE_COMMENT=106
+EXPR_WS=107
+METADATA=108
+UNQUOTED_SOURCE=109
+FROM_LINE_COMMENT=110
+FROM_MULTILINE_COMMENT=111
+FROM_WS=112
+FORK_WS=113
+FORK_LINE_COMMENT=114
+FORK_MULTILINE_COMMENT=115
+GROUP=116
+SCORE=117
+KEY=118
+FUSE_LINE_COMMENT=119
+FUSE_MULTILINE_COMMENT=120
+FUSE_WS=121
+INLINE_STATS=122
+INLINE_LINE_COMMENT=123
+INLINE_MULTILINE_COMMENT=124
+INLINE_WS=125
+JOIN=126
+USING=127
+JOIN_LINE_COMMENT=128
+JOIN_MULTILINE_COMMENT=129
+JOIN_WS=130
+LOOKUP_LINE_COMMENT=131
+LOOKUP_MULTILINE_COMMENT=132
+LOOKUP_WS=133
+LOOKUP_FIELD_LINE_COMMENT=134
+LOOKUP_FIELD_MULTILINE_COMMENT=135
+LOOKUP_FIELD_WS=136
+MMR_LIMIT=137
+MMR_LINE_COMMENT=138
+MMR_MULTILINE_COMMENT=139
+MMR_WS=140
+MVEXPAND_LINE_COMMENT=141
+MVEXPAND_MULTILINE_COMMENT=142
+MVEXPAND_WS=143
+ID_PATTERN=144
+PROJECT_LINE_COMMENT=145
+PROJECT_MULTILINE_COMMENT=146
+PROJECT_WS=147
+PROMQL_PARAMS_LINE_COMMENT=148
+PROMQL_PARAMS_MULTILINE_COMMENT=149
+PROMQL_PARAMS_WS=150
+PROMQL_QUERY_COMMENT=151
+PROMQL_SINGLE_QUOTED_STRING=152
+PROMQL_OTHER_QUERY_CONTENT=153
+AS=154
+RENAME_LINE_COMMENT=155
+RENAME_MULTILINE_COMMENT=156
+RENAME_WS=157
+SET_LINE_COMMENT=158
+SET_MULTILINE_COMMENT=159
+SET_WS=160
+INFO=161
+SHOW_LINE_COMMENT=162
+SHOW_MULTILINE_COMMENT=163
+SHOW_WS=164
'change_point'=4
'enrich'=5
'completion'=7
@@ -175,66 +176,66 @@ SHOW_WS=163
'where'=17
'from'=18
'ts'=19
-'fork'=20
-'fuse'=21
-'inline'=22
-'inlinestats'=23
-'lookup'=24
-'mv_expand'=30
-'drop'=31
-'keep'=32
-'promql'=34
-'rename'=35
-'set'=36
-'show'=37
-'|'=52
-'and'=56
-'asc'=57
-'='=58
-'by'=59
-'::'=60
-':'=61
-';'=62
-','=63
-'desc'=64
-'.'=65
-'false'=66
-'first'=67
-'in'=68
-'is'=69
-'last'=70
-'like'=71
-'not'=72
-'null'=73
-'nulls'=74
-'on'=75
-'or'=76
-'?'=77
-'rlike'=78
-'true'=79
-'with'=80
-'=='=81
-'=~'=82
-'!='=83
-'<'=84
-'<='=85
-'>'=86
-'>='=87
-'+'=88
-'-'=89
-'*'=90
-'/'=91
-'%'=92
-'{'=93
-'}'=94
-'??'=95
-']'=99
-')'=101
-'metadata'=107
-'group'=115
-'score'=116
-'key'=117
-'join'=125
-'USING'=126
-'as'=153
-'info'=160
+'fork'=21
+'fuse'=22
+'inline'=23
+'inlinestats'=24
+'lookup'=25
+'mv_expand'=31
+'drop'=32
+'keep'=33
+'promql'=35
+'rename'=36
+'set'=37
+'show'=38
+'|'=53
+'and'=57
+'asc'=58
+'='=59
+'by'=60
+'::'=61
+':'=62
+';'=63
+','=64
+'desc'=65
+'.'=66
+'false'=67
+'first'=68
+'in'=69
+'is'=70
+'last'=71
+'like'=72
+'not'=73
+'null'=74
+'nulls'=75
+'on'=76
+'or'=77
+'?'=78
+'rlike'=79
+'true'=80
+'with'=81
+'=='=82
+'=~'=83
+'!='=84
+'<'=85
+'<='=86
+'>'=87
+'>='=88
+'+'=89
+'-'=90
+'*'=91
+'/'=92
+'%'=93
+'{'=94
+'}'=95
+'??'=96
+']'=100
+')'=102
+'metadata'=108
+'group'=116
+'score'=117
+'key'=118
+'join'=126
+'USING'=127
+'as'=154
+'info'=161
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
index b10d81284dacc..a1222a46b2a6c 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -45,6 +45,7 @@ sourceCommand
| promqlCommand
// in development
| {this.isDevVersion()}? explainCommand
+ | {this.isDevVersion()}? externalCommand
;
processingCommand
@@ -102,6 +103,10 @@ timeSeriesCommand
: TS indexPatternAndMetadataFields
;
+externalCommand
+ : EXTERNAL stringOrParameter commandNamedParameters
+ ;
+
indexPatternAndMetadataFields
: indexPatternOrSubquery (COMMA indexPatternOrSubquery)* metadata?
;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
index d7837af8eea10..2bb1a5499bd79 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
@@ -17,150 +17,151 @@ STATS=16
WHERE=17
FROM=18
TS=19
-FORK=20
-FUSE=21
-INLINE=22
-INLINESTATS=23
-JOIN_LOOKUP=24
-DEV_JOIN_FULL=25
-DEV_JOIN_LEFT=26
-DEV_JOIN_RIGHT=27
-DEV_LOOKUP=28
-DEV_MMR=29
-MV_EXPAND=30
-DROP=31
-KEEP=32
-DEV_INSIST=33
-PROMQL=34
-RENAME=35
-SET=36
-SHOW=37
-UNKNOWN_CMD=38
-CHANGE_POINT_LINE_COMMENT=39
-CHANGE_POINT_MULTILINE_COMMENT=40
-CHANGE_POINT_WS=41
-ENRICH_POLICY_NAME=42
-ENRICH_LINE_COMMENT=43
-ENRICH_MULTILINE_COMMENT=44
-ENRICH_WS=45
-ENRICH_FIELD_LINE_COMMENT=46
-ENRICH_FIELD_MULTILINE_COMMENT=47
-ENRICH_FIELD_WS=48
-EXPLAIN_WS=49
-EXPLAIN_LINE_COMMENT=50
-EXPLAIN_MULTILINE_COMMENT=51
-PIPE=52
-QUOTED_STRING=53
-INTEGER_LITERAL=54
-DECIMAL_LITERAL=55
-AND=56
-ASC=57
-ASSIGN=58
-BY=59
-CAST_OP=60
-COLON=61
-SEMICOLON=62
-COMMA=63
-DESC=64
-DOT=65
-FALSE=66
-FIRST=67
-IN=68
-IS=69
-LAST=70
-LIKE=71
-NOT=72
-NULL=73
-NULLS=74
-ON=75
-OR=76
-PARAM=77
-RLIKE=78
-TRUE=79
-WITH=80
-EQ=81
-CIEQ=82
-NEQ=83
-LT=84
-LTE=85
-GT=86
-GTE=87
-PLUS=88
-MINUS=89
-ASTERISK=90
-SLASH=91
-PERCENT=92
-LEFT_BRACES=93
-RIGHT_BRACES=94
-DOUBLE_PARAMS=95
-NAMED_OR_POSITIONAL_PARAM=96
-NAMED_OR_POSITIONAL_DOUBLE_PARAMS=97
-OPENING_BRACKET=98
-CLOSING_BRACKET=99
-LP=100
-RP=101
-UNQUOTED_IDENTIFIER=102
-QUOTED_IDENTIFIER=103
-EXPR_LINE_COMMENT=104
-EXPR_MULTILINE_COMMENT=105
-EXPR_WS=106
-METADATA=107
-UNQUOTED_SOURCE=108
-FROM_LINE_COMMENT=109
-FROM_MULTILINE_COMMENT=110
-FROM_WS=111
-FORK_WS=112
-FORK_LINE_COMMENT=113
-FORK_MULTILINE_COMMENT=114
-GROUP=115
-SCORE=116
-KEY=117
-FUSE_LINE_COMMENT=118
-FUSE_MULTILINE_COMMENT=119
-FUSE_WS=120
-INLINE_STATS=121
-INLINE_LINE_COMMENT=122
-INLINE_MULTILINE_COMMENT=123
-INLINE_WS=124
-JOIN=125
-USING=126
-JOIN_LINE_COMMENT=127
-JOIN_MULTILINE_COMMENT=128
-JOIN_WS=129
-LOOKUP_LINE_COMMENT=130
-LOOKUP_MULTILINE_COMMENT=131
-LOOKUP_WS=132
-LOOKUP_FIELD_LINE_COMMENT=133
-LOOKUP_FIELD_MULTILINE_COMMENT=134
-LOOKUP_FIELD_WS=135
-MMR_LIMIT=136
-MMR_LINE_COMMENT=137
-MMR_MULTILINE_COMMENT=138
-MMR_WS=139
-MVEXPAND_LINE_COMMENT=140
-MVEXPAND_MULTILINE_COMMENT=141
-MVEXPAND_WS=142
-ID_PATTERN=143
-PROJECT_LINE_COMMENT=144
-PROJECT_MULTILINE_COMMENT=145
-PROJECT_WS=146
-PROMQL_PARAMS_LINE_COMMENT=147
-PROMQL_PARAMS_MULTILINE_COMMENT=148
-PROMQL_PARAMS_WS=149
-PROMQL_QUERY_COMMENT=150
-PROMQL_SINGLE_QUOTED_STRING=151
-PROMQL_OTHER_QUERY_CONTENT=152
-AS=153
-RENAME_LINE_COMMENT=154
-RENAME_MULTILINE_COMMENT=155
-RENAME_WS=156
-SET_LINE_COMMENT=157
-SET_MULTILINE_COMMENT=158
-SET_WS=159
-INFO=160
-SHOW_LINE_COMMENT=161
-SHOW_MULTILINE_COMMENT=162
-SHOW_WS=163
+EXTERNAL=20
+FORK=21
+FUSE=22
+INLINE=23
+INLINESTATS=24
+JOIN_LOOKUP=25
+DEV_JOIN_FULL=26
+DEV_JOIN_LEFT=27
+DEV_JOIN_RIGHT=28
+DEV_LOOKUP=29
+DEV_MMR=30
+MV_EXPAND=31
+DROP=32
+KEEP=33
+DEV_INSIST=34
+PROMQL=35
+RENAME=36
+SET=37
+SHOW=38
+UNKNOWN_CMD=39
+CHANGE_POINT_LINE_COMMENT=40
+CHANGE_POINT_MULTILINE_COMMENT=41
+CHANGE_POINT_WS=42
+ENRICH_POLICY_NAME=43
+ENRICH_LINE_COMMENT=44
+ENRICH_MULTILINE_COMMENT=45
+ENRICH_WS=46
+ENRICH_FIELD_LINE_COMMENT=47
+ENRICH_FIELD_MULTILINE_COMMENT=48
+ENRICH_FIELD_WS=49
+EXPLAIN_WS=50
+EXPLAIN_LINE_COMMENT=51
+EXPLAIN_MULTILINE_COMMENT=52
+PIPE=53
+QUOTED_STRING=54
+INTEGER_LITERAL=55
+DECIMAL_LITERAL=56
+AND=57
+ASC=58
+ASSIGN=59
+BY=60
+CAST_OP=61
+COLON=62
+SEMICOLON=63
+COMMA=64
+DESC=65
+DOT=66
+FALSE=67
+FIRST=68
+IN=69
+IS=70
+LAST=71
+LIKE=72
+NOT=73
+NULL=74
+NULLS=75
+ON=76
+OR=77
+PARAM=78
+RLIKE=79
+TRUE=80
+WITH=81
+EQ=82
+CIEQ=83
+NEQ=84
+LT=85
+LTE=86
+GT=87
+GTE=88
+PLUS=89
+MINUS=90
+ASTERISK=91
+SLASH=92
+PERCENT=93
+LEFT_BRACES=94
+RIGHT_BRACES=95
+DOUBLE_PARAMS=96
+NAMED_OR_POSITIONAL_PARAM=97
+NAMED_OR_POSITIONAL_DOUBLE_PARAMS=98
+OPENING_BRACKET=99
+CLOSING_BRACKET=100
+LP=101
+RP=102
+UNQUOTED_IDENTIFIER=103
+QUOTED_IDENTIFIER=104
+EXPR_LINE_COMMENT=105
+EXPR_MULTILINE_COMMENT=106
+EXPR_WS=107
+METADATA=108
+UNQUOTED_SOURCE=109
+FROM_LINE_COMMENT=110
+FROM_MULTILINE_COMMENT=111
+FROM_WS=112
+FORK_WS=113
+FORK_LINE_COMMENT=114
+FORK_MULTILINE_COMMENT=115
+GROUP=116
+SCORE=117
+KEY=118
+FUSE_LINE_COMMENT=119
+FUSE_MULTILINE_COMMENT=120
+FUSE_WS=121
+INLINE_STATS=122
+INLINE_LINE_COMMENT=123
+INLINE_MULTILINE_COMMENT=124
+INLINE_WS=125
+JOIN=126
+USING=127
+JOIN_LINE_COMMENT=128
+JOIN_MULTILINE_COMMENT=129
+JOIN_WS=130
+LOOKUP_LINE_COMMENT=131
+LOOKUP_MULTILINE_COMMENT=132
+LOOKUP_WS=133
+LOOKUP_FIELD_LINE_COMMENT=134
+LOOKUP_FIELD_MULTILINE_COMMENT=135
+LOOKUP_FIELD_WS=136
+MMR_LIMIT=137
+MMR_LINE_COMMENT=138
+MMR_MULTILINE_COMMENT=139
+MMR_WS=140
+MVEXPAND_LINE_COMMENT=141
+MVEXPAND_MULTILINE_COMMENT=142
+MVEXPAND_WS=143
+ID_PATTERN=144
+PROJECT_LINE_COMMENT=145
+PROJECT_MULTILINE_COMMENT=146
+PROJECT_WS=147
+PROMQL_PARAMS_LINE_COMMENT=148
+PROMQL_PARAMS_MULTILINE_COMMENT=149
+PROMQL_PARAMS_WS=150
+PROMQL_QUERY_COMMENT=151
+PROMQL_SINGLE_QUOTED_STRING=152
+PROMQL_OTHER_QUERY_CONTENT=153
+AS=154
+RENAME_LINE_COMMENT=155
+RENAME_MULTILINE_COMMENT=156
+RENAME_WS=157
+SET_LINE_COMMENT=158
+SET_MULTILINE_COMMENT=159
+SET_WS=160
+INFO=161
+SHOW_LINE_COMMENT=162
+SHOW_MULTILINE_COMMENT=163
+SHOW_WS=164
'change_point'=4
'enrich'=5
'completion'=7
@@ -175,66 +176,66 @@ SHOW_WS=163
'where'=17
'from'=18
'ts'=19
-'fork'=20
-'fuse'=21
-'inline'=22
-'inlinestats'=23
-'lookup'=24
-'mv_expand'=30
-'drop'=31
-'keep'=32
-'promql'=34
-'rename'=35
-'set'=36
-'show'=37
-'|'=52
-'and'=56
-'asc'=57
-'='=58
-'by'=59
-'::'=60
-':'=61
-';'=62
-','=63
-'desc'=64
-'.'=65
-'false'=66
-'first'=67
-'in'=68
-'is'=69
-'last'=70
-'like'=71
-'not'=72
-'null'=73
-'nulls'=74
-'on'=75
-'or'=76
-'?'=77
-'rlike'=78
-'true'=79
-'with'=80
-'=='=81
-'=~'=82
-'!='=83
-'<'=84
-'<='=85
-'>'=86
-'>='=87
-'+'=88
-'-'=89
-'*'=90
-'/'=91
-'%'=92
-'{'=93
-'}'=94
-'??'=95
-']'=99
-')'=101
-'metadata'=107
-'group'=115
-'score'=116
-'key'=117
-'join'=125
-'USING'=126
-'as'=153
-'info'=160
+'fork'=21
+'fuse'=22
+'inline'=23
+'inlinestats'=24
+'lookup'=25
+'mv_expand'=31
+'drop'=32
+'keep'=33
+'promql'=35
+'rename'=36
+'set'=37
+'show'=38
+'|'=53
+'and'=57
+'asc'=58
+'='=59
+'by'=60
+'::'=61
+':'=62
+';'=63
+','=64
+'desc'=65
+'.'=66
+'false'=67
+'first'=68
+'in'=69
+'is'=70
+'last'=71
+'like'=72
+'not'=73
+'null'=74
+'nulls'=75
+'on'=76
+'or'=77
+'?'=78
+'rlike'=79
+'true'=80
+'with'=81
+'=='=82
+'=~'=83
+'!='=84
+'<'=85
+'<='=86
+'>'=87
+'>='=88
+'+'=89
+'-'=90
+'*'=91
+'/'=92
+'%'=93
+'{'=94
+'}'=95
+'??'=96
+']'=100
+')'=102
+'metadata'=108
+'group'=116
+'score'=117
+'key'=118
+'join'=126
+'USING'=127
+'as'=154
+'info'=161
diff --git a/x-pack/plugin/esql/src/main/antlr/lexer/From.g4 b/x-pack/plugin/esql/src/main/antlr/lexer/From.g4
index 025b2055361d9..26988ededf0e5 100644
--- a/x-pack/plugin/esql/src/main/antlr/lexer/From.g4
+++ b/x-pack/plugin/esql/src/main/antlr/lexer/From.g4
@@ -14,6 +14,9 @@ FROM : 'from' -> pushMode(FROM_MODE);
// TS command
TS : 'ts' -> pushMode(FROM_MODE);
+// EXTERNAL command (development only)
+EXTERNAL : {this.isDevVersion()}? 'external' -> pushMode(FROM_MODE);
+
mode FROM_MODE;
FROM_PIPE : PIPE -> type(PIPE), popMode;
FROM_COLON : COLON -> type(COLON);
@@ -22,6 +25,13 @@ FROM_COMMA : COMMA -> type(COMMA);
FROM_ASSIGN : ASSIGN -> type(ASSIGN);
METADATA : 'metadata';
+// Support for EXTERNAL command WITH clause - transitions to EXPRESSION_MODE for map parsing
+FROM_WITH : WITH -> type(WITH), popMode, pushMode(EXPRESSION_MODE);
+
+// Support for EXTERNAL command parameters
+FROM_PARAM : PARAM -> type(PARAM);
+FROM_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
+
// we need this for EXPLAIN
// change to double popMode to accommodate subquerys in FROM, when see ')' pop out of subquery(default) mode and from mode
FROM_RP : RP -> type(RP), popMode, popMode;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 97b4f470e598b..ba3d379721bbd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -126,6 +126,7 @@
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.ExternalRelation;
import org.elasticsearch.xpack.esql.plan.logical.Fork;
import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
import org.elasticsearch.xpack.esql.plan.logical.Insist;
@@ -139,6 +140,7 @@
import org.elasticsearch.xpack.esql.plan.logical.Rename;
import org.elasticsearch.xpack.esql.plan.logical.TimeSeriesAggregate;
import org.elasticsearch.xpack.esql.plan.logical.UnionAll;
+import org.elasticsearch.xpack.esql.plan.logical.UnresolvedExternalRelation;
import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse;
import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval;
@@ -226,6 +228,7 @@ public class Analyzer extends ParameterizedRuleExecutor list, Source source, Str
}
}
+ /**
+ * Resolves UnresolvedExternalRelation nodes using pre-resolved metadata from ExternalSourceResolver.
+ * This rule mirrors the ResolveTable pattern but uses ExternalSourceResolution instead of IndexResolution.
+ *
+ * This rule creates {@link ExternalRelation} nodes from any SourceMetadata,
+ * avoiding the need for source-specific logical plan nodes in core ESQL code.
+ */
+ private static class ResolveExternalRelations extends ParameterizedAnalyzerRule {
+
+ @Override
+ protected LogicalPlan rule(UnresolvedExternalRelation plan, AnalyzerContext context) {
+ // Extract the table path from the expression
+ String tablePath = extractTablePath(plan.tablePath());
+ if (tablePath == null) {
+ // Path is not a simple literal (e.g., it's a parameter reference)
+ // Return the plan as-is for now
+ return plan;
+ }
+
+ // Get pre-resolved source (metadata + file set) from context
+ var resolvedSource = context.externalSourceResolution().get(tablePath);
+ if (resolvedSource == null) {
+ // Still unresolved - return as-is to keep the error message
+ return plan;
+ }
+
+ var metadata = resolvedSource.metadata();
+ return new ExternalRelation(plan.source(), tablePath, metadata, metadata.schema(), resolvedSource.fileSet());
+ }
+
+ private String extractTablePath(Expression tablePath) {
+ if (tablePath instanceof Literal literal && literal.value() != null) {
+ Object value = literal.value();
+ if (value instanceof org.apache.lucene.util.BytesRef) {
+ return BytesRefs.toString((org.apache.lucene.util.BytesRef) value);
+ }
+ return value.toString();
+ }
+ return null;
+ }
+ }
+
private static class ResolveEnrich extends ParameterizedAnalyzerRule {
@Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
index 86c7501547d6c..9286c1db7a5e9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
@@ -11,6 +11,7 @@
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
+import org.elasticsearch.xpack.esql.datasources.ExternalSourceResolution;
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.index.IndexResolution;
import org.elasticsearch.xpack.esql.inference.InferenceResolution;
@@ -30,6 +31,7 @@ public class AnalyzerContext {
private final Map lookupResolution;
private final EnrichResolution enrichResolution;
private final InferenceResolution inferenceResolution;
+ private final ExternalSourceResolution externalSourceResolution;
private final TransportVersion minimumVersion;
private final ProjectMetadata projectMetadata;
private Boolean hasRemoteIndices;
@@ -43,6 +45,7 @@ public AnalyzerContext(
Map lookupResolution,
EnrichResolution enrichResolution,
InferenceResolution inferenceResolution,
+ ExternalSourceResolution externalSourceResolution,
TransportVersion minimumVersion,
UnmappedResolution unmappedResolution
) {
@@ -53,6 +56,7 @@ public AnalyzerContext(
this.lookupResolution = lookupResolution;
this.enrichResolution = enrichResolution;
this.inferenceResolution = inferenceResolution;
+ this.externalSourceResolution = externalSourceResolution;
this.minimumVersion = minimumVersion;
this.unmappedResolution = unmappedResolution;
@@ -80,6 +84,7 @@ public AnalyzerContext(
lookupResolution,
enrichResolution,
inferenceResolution,
+ ExternalSourceResolution.EMPTY,
minimumVersion,
unmappedResolution
);
@@ -109,6 +114,10 @@ public InferenceResolution inferenceResolution() {
return inferenceResolution;
}
+ public ExternalSourceResolution externalSourceResolution() {
+ return externalSourceResolution;
+ }
+
public TransportVersion minimumVersion() {
return minimumVersion;
}
@@ -164,6 +173,7 @@ public AnalyzerContext(
result.lookupIndices(),
result.enrichResolution(),
result.inferenceResolution(),
+ result.externalSourceResolution(),
result.minimumTransportVersion(),
unmappedResolution
);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
index 13419894ffc50..127625766fe6b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
@@ -8,11 +8,13 @@
package org.elasticsearch.xpack.esql.analysis;
import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.util.Holder;
import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.esql.plan.IndexPattern;
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.UnresolvedExternalRelation;
import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
import java.util.ArrayList;
@@ -30,9 +32,10 @@ public record PreAnalysis(
List enriches,
List lookupIndices,
boolean useAggregateMetricDoubleWhenNotSupported,
- boolean useDenseVectorWhenNotSupported
+ boolean useDenseVectorWhenNotSupported,
+ List icebergPaths
) {
- public static final PreAnalysis EMPTY = new PreAnalysis(Map.of(), List.of(), List.of(), false, false);
+ public static final PreAnalysis EMPTY = new PreAnalysis(Map.of(), List.of(), List.of(), false, false, List.of());
}
public PreAnalysis preAnalyze(LogicalPlan plan) {
@@ -63,6 +66,18 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
List