Add connector_name to metadata.catalogs table

Author: pratyakshsharma

presto-docs/src/main/sphinx/connector/system.rst

@@ -36,7 +36,40 @@ System Connector Tables
 ``metadata.catalogs``
 ^^^^^^^^^^^^^^^^^^^^^
 
-The catalogs table contains the list of available catalogs.
+The catalogs table contains the list of available catalogs. The columns in ``metadata.catalogs`` are:
+
+======================================= ======================================================================
+Column Name                             Description
+======================================= ======================================================================
+``catalog_name``                        The value of this column is derived from the names of
+                                        catalog.properties files present under ``etc/catalog`` path under
+                                        presto installation directory. Everything except the suffix
+                                        ``.properties`` is treated as the catalog name. For example, if there
+                                        is a file named ``my_catalog.properties``, then ``my_catalog`` will be
+                                        listed as the value for this column.
+
+``connector_id``                        The values in this column are a duplicate of the values in the
+                                        ``catalog_name`` column.
+
+``connector_name``                      This column represents the actual name of the underlying connector
+                                        that a particular catalog is using. This column contains the value of
+                                        ``connector.name`` property from the catalog.properties file.
+======================================= ======================================================================
+
+Example:
+
+Suppose a user configures a single catalog by creating a file named ``my_catalog.properties`` with the
+below contents::
+
+    connector.name=hive-hadoop2
+    hive.metastore.uri=thrift://localhost:9083
+
+``metadata.catalogs`` table will show below output::
+
+    presto> select * from system.metadata.catalogs;
+     catalog_name | connector_id | connector_name
+    --------------+--------------+----------------
+     my_catalog   | my_catalog   | hive-hadoop2
 
 ``metadata.schema_properties``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java

@@ -215,10 +215,10 @@ public synchronized ConnectorId createConnection(String catalogName, String conn
         requireNonNull(connectorName, "connectorName is null");
         ConnectorFactory connectorFactory = connectorFactories.get(connectorName);
         checkArgument(connectorFactory != null, "No factory for connector %s", connectorName);
-        return createConnection(catalogName, connectorFactory, properties);
+        return createConnection(catalogName, connectorFactory, properties, connectorName);
     }
 
-    private synchronized ConnectorId createConnection(String catalogName, ConnectorFactory connectorFactory, Map<String, String> properties)
+    private synchronized ConnectorId createConnection(String catalogName, ConnectorFactory connectorFactory, Map<String, String> properties, String connectorName)
     {
         checkState(!stopped.get(), "ConnectorManager is stopped");
         requireNonNull(catalogName, "catalogName is null");
@@ -229,12 +229,12 @@ private synchronized ConnectorId createConnection(String catalogName, ConnectorF
         ConnectorId connectorId = new ConnectorId(catalogName);
         checkState(!connectors.containsKey(connectorId), "A connector %s already exists", connectorId);
 
-        addCatalogConnector(catalogName, connectorId, connectorFactory, properties);
+        addCatalogConnector(catalogName, connectorId, connectorFactory, properties, connectorName);
 
         return connectorId;
     }
 
-    private synchronized void addCatalogConnector(String catalogName, ConnectorId connectorId, ConnectorFactory factory, Map<String, String> properties)
+    private synchronized void addCatalogConnector(String catalogName, ConnectorId connectorId, ConnectorFactory factory, Map<String, String> properties, String connectorName)
     {
         // create all connectors before adding, so a broken connector does not leave the system half updated
         MaterializedConnector connector = new MaterializedConnector(connectorId, createConnector(connectorId, factory, properties));
@@ -269,7 +269,8 @@ private synchronized void addCatalogConnector(String catalogName, ConnectorId co
                 informationSchemaConnector.getConnectorId(),
                 informationSchemaConnector.getConnector(),
                 systemConnector.getConnectorId(),
-                systemConnector.getConnector());
+                systemConnector.getConnector(),
+                connectorName);
 
         try {
             addConnectorInternal(connector);

presto-main-base/src/main/java/com/facebook/presto/connector/system/CatalogSystemTable.java

@@ -15,8 +15,8 @@
 
 import com.facebook.presto.Session;
 import com.facebook.presto.common.predicate.TupleDomain;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.metadata.Metadata;
-import com.facebook.presto.spi.ConnectorId;
 import com.facebook.presto.spi.ConnectorSession;
 import com.facebook.presto.spi.ConnectorTableMetadata;
 import com.facebook.presto.spi.InMemoryRecordSet;
@@ -32,7 +32,7 @@
 
 import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType;
 import static com.facebook.presto.connector.system.SystemConnectorSessionUtil.toSession;
-import static com.facebook.presto.metadata.MetadataListing.listCatalogs;
+import static com.facebook.presto.metadata.MetadataListing.listCatalogsWithConnectorContext;
 import static com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder;
 import static com.facebook.presto.spi.SystemTable.Distribution.SINGLE_COORDINATOR;
 import static java.util.Objects.requireNonNull;
@@ -45,6 +45,7 @@ public class CatalogSystemTable
     public static final ConnectorTableMetadata CATALOG_TABLE = tableMetadataBuilder(CATALOG_TABLE_NAME)
             .column("catalog_name", createUnboundedVarcharType())
             .column("connector_id", createUnboundedVarcharType())
+            .column("connector_name", createUnboundedVarcharType())
             .build();
     private final Metadata metadata;
     private final AccessControl accessControl;
@@ -73,8 +74,8 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect
     {
         Session session = toSession(transactionHandle, connectorSession);
         Builder table = InMemoryRecordSet.builder(CATALOG_TABLE);
-        for (Map.Entry<String, ConnectorId> entry : listCatalogs(session, metadata, accessControl).entrySet()) {
-            table.addRow(entry.getKey(), entry.getValue().toString());
+        for (Map.Entry<String, CatalogContext> entry : listCatalogsWithConnectorContext(session, metadata, accessControl).entrySet()) {
+            table.addRow(entry.getKey(), entry.getValue().getCatalogName(), entry.getValue().getConnectorName());
         }
         return table.build().cursor();
     }

presto-main-base/src/main/java/com/facebook/presto/metadata/Catalog.java

@@ -32,6 +32,8 @@ public class Catalog
     private final ConnectorId systemTablesId;
     private final Connector systemTables;
 
+    private final CatalogContext catalogContext;
+
     public Catalog(
             String catalogName,
             ConnectorId connectorId,
@@ -40,6 +42,26 @@ public Catalog(
             Connector informationSchema,
             ConnectorId systemTablesId,
             Connector systemTables)
+    {
+        this(catalogName,
+                connectorId,
+                connector,
+                informationSchemaId,
+                informationSchema,
+                systemTablesId,
+                systemTables,
+                catalogName);
+    }
+
+    public Catalog(
+            String catalogName,
+            ConnectorId connectorId,
+            Connector connector,
+            ConnectorId informationSchemaId,
+            Connector informationSchema,
+            ConnectorId systemTablesId,
+            Connector systemTables,
+            String connectorName)
     {
         this.catalogName = checkCatalogName(catalogName);
         this.connectorId = requireNonNull(connectorId, "connectorId is null");
@@ -48,8 +70,9 @@ public Catalog(
         this.informationSchema = requireNonNull(informationSchema, "informationSchema is null");
         this.systemTablesId = requireNonNull(systemTablesId, "systemTablesId is null");
         this.systemTables = requireNonNull(systemTables, "systemTables is null");
+        requireNonNull(connectorName, "connectorName is null");
+        this.catalogContext = new CatalogContext(catalogName, connectorName);
     }
-
     public String getCatalogName()
     {
         return catalogName;
@@ -60,6 +83,11 @@ public ConnectorId getConnectorId()
         return connectorId;
     }
 
+    public CatalogContext getCatalogContext()
+    {
+        return catalogContext;
+    }
+
     public ConnectorId getInformationSchemaId()
     {
         return informationSchemaId;
@@ -92,4 +120,26 @@ public String toString()
                 .add("connectorId", connectorId)
                 .toString();
     }
+
+    public class CatalogContext
+    {
+        private final String catalogName;
+        private final String connectorName;
+
+        public CatalogContext(String catalogName, String connectorName)
+        {
+            this.catalogName = catalogName;
+            this.connectorName = connectorName;
+        }
+
+        public String getCatalogName()
+        {
+            return catalogName;
+        }
+
+        public String getConnectorName()
+        {
+            return connectorName;
+        }
+    }
 }

presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java

@@ -20,6 +20,7 @@
 import com.facebook.presto.common.predicate.TupleDomain;
 import com.facebook.presto.common.type.Type;
 import com.facebook.presto.common.type.TypeSignature;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.spi.ColumnHandle;
 import com.facebook.presto.spi.ColumnMetadata;
 import com.facebook.presto.spi.ConnectorId;
@@ -52,6 +53,7 @@
 import com.facebook.presto.spi.statistics.ComputedStatistics;
 import com.facebook.presto.spi.statistics.TableStatistics;
 import com.facebook.presto.spi.statistics.TableStatisticsMetadata;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
 import io.airlift.slice.Slice;
 
@@ -360,6 +362,11 @@ public interface Metadata
      */
     Map<String, ConnectorId> getCatalogNames(Session session);
 
+    default Map<String, CatalogContext> getCatalogNamesWithConnectorContext(Session session)
+    {
+        return ImmutableMap.of();
+    }
+
     /**
      * Get the names that match the specified table prefix (never null).
      */

presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataListing.java

@@ -15,6 +15,7 @@
 
 import com.facebook.presto.Session;
 import com.facebook.presto.common.QualifiedObjectName;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.spi.ColumnMetadata;
 import com.facebook.presto.spi.ConnectorId;
 import com.facebook.presto.spi.SchemaTableName;
@@ -54,6 +55,20 @@ public static SortedMap<String, ConnectorId> listCatalogs(Session session, Metad
         return result.build();
     }
 
+    public static SortedMap<String, CatalogContext> listCatalogsWithConnectorContext(Session session, Metadata metadata, AccessControl accessControl)
+    {
+        Map<String, CatalogContext> catalogNamesWithConnectorContext = metadata.getCatalogNamesWithConnectorContext(session);
+        Set<String> allowedCatalogs = accessControl.filterCatalogs(session.getIdentity(), session.getAccessControlContext(), catalogNamesWithConnectorContext.keySet());
+
+        ImmutableSortedMap.Builder<String, CatalogContext> result = ImmutableSortedMap.naturalOrder();
+        for (Map.Entry<String, CatalogContext> entry : catalogNamesWithConnectorContext.entrySet()) {
+            if (allowedCatalogs.contains(entry.getKey())) {
+                result.put(entry);
+            }
+        }
+        return result.build();
+    }
+
     public static SortedSet<String> listSchemas(Session session, Metadata metadata, AccessControl accessControl, String catalogName)
     {
         Set<String> schemaNames = ImmutableSet.copyOf(metadata.listSchemaNames(session, catalogName));

presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java

@@ -26,6 +26,7 @@
 import com.facebook.presto.common.predicate.TupleDomain;
 import com.facebook.presto.common.type.Type;
 import com.facebook.presto.common.type.TypeSignature;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.spi.ColumnHandle;
 import com.facebook.presto.spi.ColumnMetadata;
 import com.facebook.presto.spi.ConnectorDeleteTableHandle;
@@ -987,6 +988,12 @@ public Map<String, ConnectorId> getCatalogNames(Session session)
         return transactionManager.getCatalogNames(session.getRequiredTransactionId());
     }
 
+    @Override
+    public Map<String, CatalogContext> getCatalogNamesWithConnectorContext(Session session)
+    {
+        return transactionManager.getCatalogNamesWithConnectorContext(session.getRequiredTransactionId());
+    }
+
     @Override
     public List<QualifiedObjectName> listViews(Session session, QualifiedTablePrefix prefix)
     {

presto-main-base/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java

@@ -19,6 +19,7 @@
 import com.facebook.airlift.units.Duration;
 import com.facebook.presto.common.transaction.TransactionId;
 import com.facebook.presto.metadata.Catalog;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.metadata.CatalogManager;
 import com.facebook.presto.metadata.CatalogMetadata;
 import com.facebook.presto.spi.ConnectorId;
@@ -214,6 +215,12 @@ public Map<String, ConnectorId> getCatalogNames(TransactionId transactionId)
         return getTransactionMetadata(transactionId).getCatalogNames();
     }
 
+    @Override
+    public Map<String, CatalogContext> getCatalogNamesWithConnectorContext(TransactionId transactionId)
+    {
+        return getTransactionMetadata(transactionId).getCatalogNamesWithConnectorContext();
+    }
+
     @Override
     public Optional<CatalogMetadata> getOptionalCatalogMetadata(TransactionId transactionId, String catalogName)
     {
@@ -462,6 +469,20 @@ private synchronized Map<String, ConnectorId> getCatalogNames()
             return ImmutableMap.copyOf(catalogNames);
         }
 
+        private synchronized Map<String, CatalogContext> getCatalogNamesWithConnectorContext()
+        {
+            Map<String, CatalogContext> catalogNamesWithConnectorContext = new HashMap<>();
+            catalogByName.values().stream()
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    .forEach(catalog -> catalogNamesWithConnectorContext.put(catalog.getCatalogName(), catalog.getCatalogContext()));
+
+            catalogManager.getCatalogs().stream()
+                    .forEach(catalog -> catalogNamesWithConnectorContext.putIfAbsent(catalog.getCatalogName(), catalog.getCatalogContext()));
+
+            return ImmutableMap.copyOf(catalogNamesWithConnectorContext);
+        }
+
         private synchronized Optional<ConnectorId> getConnectorId(String catalogName)
         {
             Optional<Catalog> catalog = catalogByName.get(catalogName);

presto-main-base/src/main/java/com/facebook/presto/transaction/TransactionManager.java

@@ -15,13 +15,15 @@
 
 import com.facebook.presto.Session;
 import com.facebook.presto.common.transaction.TransactionId;
+import com.facebook.presto.metadata.Catalog.CatalogContext;
 import com.facebook.presto.metadata.CatalogMetadata;
 import com.facebook.presto.spi.ConnectorId;
 import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
 import com.facebook.presto.spi.function.FunctionNamespaceManager;
 import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle;
 import com.facebook.presto.spi.security.AccessControl;
 import com.facebook.presto.spi.transaction.IsolationLevel;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
 
 import java.util.List;
@@ -47,6 +49,11 @@ public interface TransactionManager
 
     Map<String, ConnectorId> getCatalogNames(TransactionId transactionId);
 
+    default Map<String, CatalogContext> getCatalogNamesWithConnectorContext(TransactionId transactionId)
+    {
+        return ImmutableMap.of();
+    }
+
     Optional<CatalogMetadata> getOptionalCatalogMetadata(TransactionId transactionId, String catalogName);
 
     void enableRollback(TransactionId transactionId);

presto-product-tests/src/main/java/com/facebook/presto/tests/SystemConnectorTests.java

@@ -108,9 +108,9 @@ public void selectRuntimeTasks()
     @Test(groups = {SYSTEM_CONNECTOR, JDBC})
     public void selectMetadataCatalogs()
     {
-        String sql = "select catalog_name, connector_id from system.metadata.catalogs";
+        String sql = "select catalog_name, connector_id, connector_name from system.metadata.catalogs";
         assertThat(query(sql))
-                .hasColumns(VARCHAR, VARCHAR)
+                .hasColumns(VARCHAR, VARCHAR, VARCHAR)
                 .hasAnyRows();
     }