Merge branch 'master' into fd-parquet

Fokko · web-flow · commit 00a1b2ff1986 · 2025-09-02T15:00:57.000+02:00
diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java b/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java
@@ -55,7 +55,8 @@ public CatalogConnector(
                 connectorName,
                 catalogConnector,
                 informationSchemaConnector,
-                systemConnector);
+                systemConnector,
+                true);
     }
 
     public CatalogHandle getCatalogHandle()
diff --git a/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java b/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java
@@ -29,6 +29,9 @@
 import io.trino.spi.connector.SystemTable;
 import io.trino.spi.predicate.TupleDomain;
 
+import java.util.List;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static io.trino.metadata.MetadataListing.listCatalogs;
 import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder;
 import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR;
@@ -72,7 +75,10 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect
     {
         Session session = ((FullConnectorSession) connectorSession).getSession();
         Builder table = InMemoryRecordSet.builder(CATALOG_TABLE);
-        for (CatalogInfo catalogInfo : listCatalogs(session, metadata, accessControl)) {
+        List<CatalogInfo> catalogInfos = listCatalogs(session, metadata, accessControl).stream()
+                .filter(CatalogInfo::loaded)
+                .collect(toImmutableList());
+        for (CatalogInfo catalogInfo : catalogInfos) {
             table.addRow(
                     catalogInfo.catalogName(),
                     catalogInfo.catalogName(),
diff --git a/core/trino-main/src/main/java/io/trino/metadata/Catalog.java b/core/trino-main/src/main/java/io/trino/metadata/Catalog.java
@@ -38,14 +38,16 @@ public class Catalog
     private final ConnectorServices catalogConnector;
     private final ConnectorServices informationSchemaConnector;
     private final ConnectorServices systemConnector;
+    private final boolean loaded;
 
     public Catalog(
             CatalogName catalogName,
             CatalogHandle catalogHandle,
             ConnectorName connectorName,
             ConnectorServices catalogConnector,
             ConnectorServices informationSchemaConnector,
-            ConnectorServices systemConnector)
+            ConnectorServices systemConnector,
+            boolean loaded)
     {
         this.catalogName = requireNonNull(catalogName, "catalogName is null");
         this.catalogHandle = requireNonNull(catalogHandle, "catalogHandle is null");
@@ -54,21 +56,23 @@ public Catalog(
         this.catalogConnector = requireNonNull(catalogConnector, "catalogConnector is null");
         this.informationSchemaConnector = requireNonNull(informationSchemaConnector, "informationSchemaConnector is null");
         this.systemConnector = requireNonNull(systemConnector, "systemConnector is null");
+        this.loaded = loaded;
     }
 
     public static Catalog failedCatalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
     {
-        return new Catalog(catalogName, catalogHandle, connectorName);
+        return new Catalog(catalogName, catalogHandle, connectorName, false);
     }
 
-    private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
+    private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded)
     {
         this.catalogName = catalogName;
         this.catalogHandle = catalogHandle;
         this.connectorName = connectorName;
         this.catalogConnector = null;
         this.informationSchemaConnector = null;
         this.systemConnector = null;
+        this.loaded = loaded;
     }
 
     public CatalogName getCatalogName()
@@ -86,6 +90,11 @@ public ConnectorName getConnectorName()
         return connectorName;
     }
 
+    public boolean isLoaded()
+    {
+        return loaded;
+    }
+
     public boolean isFailed()
     {
         return catalogConnector == null;
diff --git a/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java b/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java
@@ -18,7 +18,7 @@
 
 import static java.util.Objects.requireNonNull;
 
-public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
+public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded)
 {
     public CatalogInfo
     {
diff --git a/core/trino-main/src/main/java/io/trino/metadata/Metadata.java b/core/trino-main/src/main/java/io/trino/metadata/Metadata.java
@@ -467,7 +467,7 @@ Optional<ConnectorOutputMetadata> finishRefreshMaterializedView(
     Optional<CatalogHandle> getCatalogHandle(Session session, String catalogName);
 
     /**
-     * Gets all the loaded catalogs
+     * Lists all defined catalogs (both loaded properly and failed ones).
      */
     List<CatalogInfo> listCatalogs(Session session);
 
diff --git a/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java b/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java
@@ -320,8 +320,9 @@ private ProcessBatchResult processBatch(int batchSize)
                     blocks[i] = previouslyComputedResults[i].getRegion(0, batchSize);
                 }
                 else {
+                    SourcePage inputChannelsSourcePage = projection.getInputChannels().getInputChannels(page);
                     expressionProfiler.start();
-                    Block result = projection.project(session, projection.getInputChannels().getInputChannels(page), positionsBatch);
+                    Block result = projection.project(session, inputChannelsSourcePage, positionsBatch);
                     long projectionTimeNanos = expressionProfiler.stop(positionsBatch.size());
                     metrics.recordProjectionTime(projectionTimeNanos);
 
diff --git a/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java b/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java
@@ -423,7 +423,7 @@ private synchronized List<CatalogInfo> getActiveCatalogs()
                     .distinct()
                     .map(key -> registeredCatalogs.getOrDefault(key, Optional.empty()))
                     .flatMap(Optional::stream)
-                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName()))
+                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded()))
                     .collect(toImmutableList());
         }
 
@@ -436,7 +436,7 @@ private synchronized List<CatalogInfo> listCatalogs()
             return registeredCatalogs.values().stream()
                     .filter(Optional::isPresent)
                     .map(Optional::get)
-                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName()))
+                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded()))
                     .collect(toImmutableList());
         }
 
diff --git a/lib/trino-geospatial-toolkit/src/main/java/io/trino/geospatial/GeometryUtils.java b/lib/trino-geospatial-toolkit/src/main/java/io/trino/geospatial/GeometryUtils.java
@@ -179,6 +179,8 @@ public static org.locationtech.jts.geom.Geometry jtsGeometryFromJson(String json
 
     public static String jsonFromJtsGeometry(org.locationtech.jts.geom.Geometry geometry)
     {
-        return new GeoJsonWriter().write(geometry);
+        GeoJsonWriter geoJsonWriter = new GeoJsonWriter();
+        geoJsonWriter.setEncodeCRS(false);
+        return geoJsonWriter.write(geometry);
     }
 }
diff --git a/lib/trino-geospatial-toolkit/src/test/java/io/trino/geospatial/TestGeometryUtils.java b/lib/trino-geospatial-toolkit/src/test/java/io/trino/geospatial/TestGeometryUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.geospatial;
+
+import org.junit.jupiter.api.Test;
+import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKTReader;
+
+import static io.trino.geospatial.GeometryUtils.jsonFromJtsGeometry;
+import static org.assertj.core.api.Assertions.assertThat;
+
+final class TestGeometryUtils
+{
+    @Test
+    void testJsonFromJtsGeometry()
+            throws ParseException
+    {
+        String json = jsonFromJtsGeometry(new WKTReader().read("POINT (1 1)"));
+        assertThat(json)
+                .isNotNull()
+                .doesNotContain("crs");
+    }
+}
diff --git a/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListener.java b/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListener.java
@@ -88,9 +88,7 @@ public OpenLineageListener(OpenLineage openLineage, OpenLineageClient client, Op
         this.jobNamespace = listenerConfig.getNamespace().orElse(trinoURI.toString());
         this.datasetNamespace = trinoURI.toString();
         this.includeQueryTypes = ImmutableSet.copyOf(listenerConfig.getIncludeQueryTypes());
-        this.interpolator = new FormatInterpolator(
-                                listenerConfig.getJobNameFormat(),
-                                OpenLineageJobInterpolatedValues.values());
+        this.interpolator = new FormatInterpolator<>(listenerConfig.getJobNameFormat(), OpenLineageJobInterpolatedValues.values());
     }
 
     @Override
@@ -291,9 +289,9 @@ private JobBuilder getBaseJobBuilder(QueryContext queryContext, QueryMetadata qu
                 .namespace(this.jobNamespace)
                 .name(interpolator.interpolate(new OpenLineageJobContext(queryContext, queryMetadata)))
                 .facets(openLineage.newJobFacetsBuilder()
-                            .jobType(openLineage.newJobTypeJobFacet("BATCH", "TRINO", "QUERY"))
-                            .sql(openLineage.newSQLJobFacet(queryMetadata.getQuery(), "trino"))
-                            .build());
+                        .jobType(openLineage.newJobTypeJobFacet("BATCH", "TRINO", "QUERY"))
+                        .sql(openLineage.newSQLJobFacet(queryMetadata.getQuery(), "trino"))
+                        .build());
     }
 
     private List<InputDataset> buildInputs(QueryMetadata queryMetadata)
diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java
@@ -31,9 +31,9 @@
 import static java.util.Objects.requireNonNull;
 
 /**
-* - TestOpenLineageEventsFromQueries requires sequential execution to properly reset memory listener
-* - TestOpenLineageEventListenerMarquezIntegration multiple parallel tests are flaky, as they increase wait time for marquez to register events
-*/
+ * - TestOpenLineageEventsFromQueries requires sequential execution to properly reset memory listener
+ * - TestOpenLineageEventListenerMarquezIntegration multiple parallel tests are flaky, as they increase wait time for marquez to register events
+ */
 @Execution(ExecutionMode.SAME_THREAD)
 public abstract class BaseTestOpenLineageQueries
         extends AbstractTestQueryFramework
diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TestOpenLineageListener.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TestOpenLineageListener.java
@@ -53,7 +53,7 @@ void testGetCompleteEvent()
         assertThat(result)
                 .extracting(RunEvent::getRun)
                 .extracting(Run::getRunId)
-                 // random UUID part may differ, but prefix is timestamp based
+                // random UUID part may differ, but prefix is timestamp based
                 .matches(uuid -> uuid.toString().startsWith("01967c23-ae78-7"));
 
         assertThat(result)
@@ -66,8 +66,7 @@ void testGetCompleteEvent()
                 .extracting(Job::getName)
                 .isEqualTo("queryId");
 
-        Map<String, Object> trinoQueryMetadata =
-                result
+        Map<String, Object> trinoQueryMetadata = result
                 .getRun()
                 .getFacets()
                 .getAdditionalProperties()
@@ -82,11 +81,11 @@ void testGetCompleteEvent()
 
         Map<String, Object> trinoQueryContext =
                 result
-                .getRun()
-                .getFacets()
-                .getAdditionalProperties()
-                .get("trino_query_context")
-                .getAdditionalProperties();
+                        .getRun()
+                        .getFacets()
+                        .getAdditionalProperties()
+                        .get("trino_query_context")
+                        .getAdditionalProperties();
 
         assertThat(trinoQueryContext)
                 .containsOnly(
@@ -124,7 +123,7 @@ void testGetStartEvent()
         assertThat(result)
                 .extracting(RunEvent::getRun)
                 .extracting(Run::getRunId)
-                 // random UUID part may differ, but prefix is timestamp based
+                // random UUID part may differ, but prefix is timestamp based
                 .matches(uuid -> uuid.toString().startsWith("01967c23-ae78-7"));
 
         assertThat(result)
@@ -137,8 +136,7 @@ void testGetStartEvent()
                 .extracting(Job::getName)
                 .isEqualTo("queryId");
 
-        Map<String, Object> trinoQueryMetadata =
-                result
+        Map<String, Object> trinoQueryMetadata = result
                 .getRun()
                 .getFacets()
                 .getAdditionalProperties()
@@ -151,8 +149,7 @@ void testGetStartEvent()
                         entry("transaction_id", "transactionId"),
                         entry("query_plan", "queryPlan"));
 
-        Map<String, Object> trinoQueryContext =
-                result
+        Map<String, Object> trinoQueryContext = result
                 .getRun()
                 .getFacets()
                 .getAdditionalProperties()
diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TestOpenLineageListenerConfig.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TestOpenLineageListenerConfig.java
@@ -125,7 +125,7 @@ void testIsJobNameFormatValid()
     private static OpenLineageListenerConfig configWithFormat(String format)
     {
         return new OpenLineageListenerConfig()
-            .setTrinoURI(URI.create("http://testtrino"))
-            .setJobNameFormat(format);
+                .setTrinoURI(URI.create("http://testtrino"))
+                .setJobNameFormat(format);
     }
 }
diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java
@@ -51,8 +51,7 @@ private TrinoEventData()
         throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
     }
 
-    static
-    {
+    static {
         queryIOMetadata = new QueryIOMetadata(Collections.emptyList(), Optional.empty());
 
         queryContext = new QueryContext(
diff --git a/plugin/trino-openlineage/src/test/resources/marquez.yaml b/plugin/trino-openlineage/src/test/resources/marquez.yaml
@@ -1,11 +1,11 @@
 server:
     applicationConnectors:
-        - type: http
-          port: ${MARQUEZ_PORT:-5000}
-          httpCompliance: RFC7230_LEGACY
+        -   type: http
+            port: ${MARQUEZ_PORT:-5000}
+            httpCompliance: RFC7230_LEGACY
     adminConnectors:
-        - type: http
-          port: ${MARQUEZ_ADMIN_PORT:-5001}
+        -   type: http
+            port: ${MARQUEZ_ADMIN_PORT:-5001}
 
 db:
     driverClass: org.postgresql.Driver
@@ -21,10 +21,10 @@ graphql:
 logging:
     level: INFO
     appenders:
-        - type: console
+        -   type: console
 
 tags:
-    - name: PII
-      description: Personally identifiable information
-    - name: SENSITIVE
-      description: Contains sensitive information
+    -   name: PII
+        description: Personally identifiable information
+    -   name: SENSITIVE
+        description: Contains sensitive information
diff --git a/pom.xml b/pom.xml
@@ -469,7 +469,7 @@
             <dependency>
                 <groupId>com.google.cloud.bigdataoss</groupId>
                 <artifactId>gcs-connector</artifactId>
-                <version>3.1.4</version>
+                <version>3.1.5</version>
                 <classifier>shaded</classifier>
                 <exclusions>
                     <exclusion>
diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java
@@ -16,16 +16,14 @@
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import io.trino.tempto.ProductTest;
-import io.trino.tempto.assertions.QueryAssert;
 import org.testng.SkipException;
 import org.testng.annotations.Test;
 
 import java.util.List;
-import java.util.stream.Collectors;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
 import static io.trino.tests.product.utils.QueryExecutors.onTrino;
-import static java.sql.JDBCType.VARCHAR;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -49,10 +47,16 @@ public void selectConfiguredConnectors()
             throw new SkipException("Skip checking configured connectors since none were set in Tempto configuration");
         }
         String sql = "SELECT DISTINCT connector_name FROM system.metadata.catalogs";
-        assertThat(onTrino().executeQuery(sql))
-                .hasColumns(VARCHAR)
-                .containsOnly(configuredConnectors.stream()
-                        .map(QueryAssert.Row::row)
-                        .collect(Collectors.toList()));
+        List<String> loadedCatalogs = onTrino().executeQuery(sql).column(1).stream()
+                .map(Object::toString)
+                .collect(toImmutableList());
+        // TODO https://github.com/trinodb/trino/issues/26500
+        // Loki connector is not loading properly. Once this is fixed, test will fail.
+        List<String> filteredCatalogs = configuredConnectors.stream()
+                .filter(connector -> !connector.equals("loki"))
+                .collect(toImmutableList());
+
+        assertThat(filteredCatalogs)
+                .containsExactlyInAnyOrder(loadedCatalogs.toArray(new String[0]));
     }
 }
diff --git a/testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java b/testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java

Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,8 @@ public CatalogConnector(`
`55`	`55`	`connectorName,`
`56`	`56`	`catalogConnector,`
`57`	`57`	`informationSchemaConnector,`
`58`		`- systemConnector);`
	`58`	`+ systemConnector,`
	`59`	`+ true);`
`59`	`60`	`}`
`60`	`61`
`61`	`62`	`public CatalogHandle getCatalogHandle()`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`
`19`	`19`	`import static java.util.Objects.requireNonNull;`
`20`	`20`
`21`		`-public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)`
	`21`	`+public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded)`
`22`	`22`	`{`
`23`	`23`	`public CatalogInfo`
`24`	`24`	`{`
Original file line number	Diff line number	Diff line change
`@@ -179,6 +179,8 @@ public static org.locationtech.jts.geom.Geometry jtsGeometryFromJson(String json`
`179`	`179`
`180`	`180`	`public static String jsonFromJtsGeometry(org.locationtech.jts.geom.Geometry geometry)`
`181`	`181`	`{`
`182`		`- return new GeoJsonWriter().write(geometry);`
	`182`	`+ GeoJsonWriter geoJsonWriter = new GeoJsonWriter();`
	`183`	`+ geoJsonWriter.setEncodeCRS(false);`
	`184`	`+ return geoJsonWriter.write(geometry);`
`183`	`185`	`}`
`184`	`186`	`}`