metadata.catalogs should only list loaded catalogs

Author: homar

core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java

@@ -55,7 +55,8 @@ public CatalogConnector(
                 connectorName,
                 catalogConnector,
                 informationSchemaConnector,
-                systemConnector);
+                systemConnector,
+                true);
     }
 
     public CatalogHandle getCatalogHandle()

core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java

@@ -29,6 +29,9 @@
 import io.trino.spi.connector.SystemTable;
 import io.trino.spi.predicate.TupleDomain;
 
+import java.util.List;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static io.trino.metadata.MetadataListing.listCatalogs;
 import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder;
 import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR;
@@ -72,7 +75,10 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect
     {
         Session session = ((FullConnectorSession) connectorSession).getSession();
         Builder table = InMemoryRecordSet.builder(CATALOG_TABLE);
-        for (CatalogInfo catalogInfo : listCatalogs(session, metadata, accessControl)) {
+        List<CatalogInfo> catalogInfos = listCatalogs(session, metadata, accessControl).stream()
+                .filter(CatalogInfo::loaded)
+                .collect(toImmutableList());
+        for (CatalogInfo catalogInfo : catalogInfos) {
             table.addRow(
                     catalogInfo.catalogName(),
                     catalogInfo.catalogName(),

core/trino-main/src/main/java/io/trino/metadata/Catalog.java

@@ -38,14 +38,16 @@ public class Catalog
     private final ConnectorServices catalogConnector;
     private final ConnectorServices informationSchemaConnector;
     private final ConnectorServices systemConnector;
+    private final boolean loaded;
 
     public Catalog(
             CatalogName catalogName,
             CatalogHandle catalogHandle,
             ConnectorName connectorName,
             ConnectorServices catalogConnector,
             ConnectorServices informationSchemaConnector,
-            ConnectorServices systemConnector)
+            ConnectorServices systemConnector,
+            boolean loaded)
     {
         this.catalogName = requireNonNull(catalogName, "catalogName is null");
         this.catalogHandle = requireNonNull(catalogHandle, "catalogHandle is null");
@@ -54,21 +56,23 @@ public Catalog(
         this.catalogConnector = requireNonNull(catalogConnector, "catalogConnector is null");
         this.informationSchemaConnector = requireNonNull(informationSchemaConnector, "informationSchemaConnector is null");
         this.systemConnector = requireNonNull(systemConnector, "systemConnector is null");
+        this.loaded = loaded;
     }
 
     public static Catalog failedCatalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
     {
-        return new Catalog(catalogName, catalogHandle, connectorName);
+        return new Catalog(catalogName, catalogHandle, connectorName, false);
     }
 
-    private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
+    private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded)
     {
         this.catalogName = catalogName;
         this.catalogHandle = catalogHandle;
         this.connectorName = connectorName;
         this.catalogConnector = null;
         this.informationSchemaConnector = null;
         this.systemConnector = null;
+        this.loaded = loaded;
     }
 
     public CatalogName getCatalogName()
@@ -86,6 +90,11 @@ public ConnectorName getConnectorName()
         return connectorName;
     }
 
+    public boolean isLoaded()
+    {
+        return loaded;
+    }
+
     public boolean isFailed()
     {
         return catalogConnector == null;

core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java

@@ -18,7 +18,11 @@
 
 import static java.util.Objects.requireNonNull;
 
-public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName)
+public record CatalogInfo(
+        String catalogName,
+        CatalogHandle catalogHandle,
+        ConnectorName connectorName,
+        boolean loaded)
 {
     public CatalogInfo
     {

core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java

@@ -423,7 +423,7 @@ private synchronized List<CatalogInfo> getActiveCatalogs()
                     .distinct()
                     .map(key -> registeredCatalogs.getOrDefault(key, Optional.empty()))
                     .flatMap(Optional::stream)
-                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName()))
+                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded()))
                     .collect(toImmutableList());
         }
 
@@ -436,7 +436,7 @@ private synchronized List<CatalogInfo> listCatalogs()
             return registeredCatalogs.values().stream()
                     .filter(Optional::isPresent)
                     .map(Optional::get)
-                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName()))
+                    .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded()))
                     .collect(toImmutableList());
         }
 

testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java

@@ -47,7 +47,13 @@ public void selectConfiguredConnectors()
             throw new SkipException("Skip checking configured connectors since none were set in Tempto configuration");
         }
         String sql = "SELECT DISTINCT connector_name FROM system.metadata.catalogs";
-        List<String> loadedCatalogs = onTrino().executeQuery(sql).column(1).stream().map(Object::toString).collect(toImmutableList());
-        assertThat(configuredConnectors).containsExactlyInAnyOrder(loadedCatalogs.toArray(new String[0]));
+        List<String> loadedCatalogs = onTrino().executeQuery(sql).column(1).stream()
+                .map(Object::toString)
+                .collect(toImmutableList());
+        // TODO https://github.com/trinodb/trino/issues/26500
+        // for now loki connector is not properly loaded, when this is fixed this test will fail
+        List<String> filtered = configuredConnectors.stream().filter(connector -> !connector.equals("loki")).collect(toImmutableList());
+        assertThat(filtered)
+                .containsExactlyInAnyOrder(loadedCatalogs.toArray(new String[0]));
     }
 }

testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.connector.system;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Key;
+import io.trino.Session;
+import io.trino.connector.ConnectorServicesProvider;
+import io.trino.plugin.memory.MemoryPlugin;
+import io.trino.spi.catalog.CatalogName;
+import io.trino.spi.catalog.CatalogProperties;
+import io.trino.spi.catalog.CatalogStore;
+import io.trino.spi.connector.ConnectorName;
+import io.trino.testing.AbstractTestQueryFramework;
+import io.trino.testing.DistributedQueryRunner;
+import io.trino.testing.QueryRunner;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Execution;
+
+import static io.trino.testing.TestingSession.testSessionBuilder;
+import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
+
+@Execution(SAME_THREAD)
+public class TestSystemMetadataCatalogTable
+        extends AbstractTestQueryFramework
+{
+    private CatalogStore catalogStore;
+    private ConnectorServicesProvider catalogManager;
+
+    @Override
+    protected QueryRunner createQueryRunner()
+            throws Exception
+    {
+        Session session = testSessionBuilder().build();
+        QueryRunner queryRunner = DistributedQueryRunner.builder(session)
+                .setWorkerCount(0)
+                .build();
+        queryRunner.installPlugin(new MemoryPlugin());
+        queryRunner.createCatalog("healthy_catalog", "memory", ImmutableMap.of("memory.max-data-per-node", "128MB"));
+        catalogStore = queryRunner.getCoordinator().getInstance(new Key<>() {});
+        catalogManager = queryRunner.getCoordinator().getInstance(new Key<>() {});
+        return queryRunner;
+    }
+
+    @Test
+    public void testCatalogTableShowsOnlyLoadedCatalogs()
+    {
+        assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" +
+                "('healthy_catalog', 'healthy_catalog', 'memory'), " +
+                "('system', 'system', 'system')");
+
+        assertUpdate("CREATE CATALOG brain USING memory WITH (\"memory.max-data-per-node\" = '128MB')");
+        assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" +
+                "('healthy_catalog', 'healthy_catalog', 'memory'), " +
+                "('brain', 'brain', 'memory'), " +
+                "('system', 'system', 'system')");
+        CatalogProperties catalogProperties = catalogStore.createCatalogProperties(new CatalogName("broken"), new ConnectorName("memory"), ImmutableMap.of("memory.max-data-per-n", "128MB"));
+        catalogStore.addOrReplaceCatalog(catalogProperties);
+        catalogManager.loadInitialCatalogs();
+
+        assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" +
+                "('healthy_catalog', 'healthy_catalog', 'memory'), " +
+                "('brain', 'brain', 'memory'), " +
+                "('system', 'system', 'system')");
+    }
+}