Add invoked functions to query completed event

presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java

@@ -900,6 +900,11 @@ public Optional<QuerySpecification> getCurrentQuerySpecification()
         return currentQuerySpecification;
     }
 
+    public List<String> getInvokedFunctionNames()
+    {
+        return ImmutableList.copyOf(functionHandles.values().stream().map(FunctionHandle::getName).collect(toImmutableSet()));
+    }
+
     @Immutable
     public static final class Insert
     {

presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java

@@ -41,6 +41,12 @@ public CatalogSchemaName getCatalogSchemaName()
         return signature.getName().getCatalogSchemaName();
     }
 
+    @Override
+    public String getName()
+    {
+        return signature.getName().toString();
+    }
+
     @JsonProperty
     public Signature getSignature()
     {

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java

@@ -85,6 +85,7 @@
 import static com.facebook.presto.SystemSessionProperties.INLINE_SQL_FUNCTIONS;
 import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
 import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
+import static com.facebook.presto.SystemSessionProperties.LOG_INVOKED_FUNCTION_NAMES_ENABLED;
 import static com.facebook.presto.SystemSessionProperties.PARTIAL_MERGE_PUSHDOWN_STRATEGY;
 import static com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG;
 import static com.facebook.presto.common.predicate.Marker.Bound.EXACTLY;
@@ -154,6 +155,7 @@
 import static java.util.stream.Collectors.joining;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.testng.Assert.assertEqualsNoOrder;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertNotNull;
@@ -5898,6 +5900,39 @@ public void testAlphaFormatDml()
         assertUpdate("DROP TABLE test_alpha_dml_partitioned_table");
     }
 
+    @Test
+    public void testInvokedFunctionNamesLog()
+    {
+        QueryRunner queryRunner = getQueryRunner();
+        Session logFunctionNamesEnabledSession = Session.builder(getSession())
+                .setSystemProperty(LOG_INVOKED_FUNCTION_NAMES_ENABLED, "true")
+                .build();
+        ResultWithQueryId<MaterializedResult> resultWithQueryId;
+        QueryInfo queryInfo;
+
+        @Language("SQL") String queryWithScalarFunctions =
+                "SELECT abs(acctbal), round(acctbal), round(acctbal, 1), repeat(custkey, 2), repeat(name, 3),  repeat(mktsegment, 4) FROM customer";
+        resultWithQueryId = ((DistributedQueryRunner) queryRunner).executeWithQueryId(logFunctionNamesEnabledSession, queryWithScalarFunctions);
+        queryInfo = ((DistributedQueryRunner) queryRunner).getQueryInfo(resultWithQueryId.getQueryId());
+        assertEqualsNoOrder(queryInfo.getFunctionNames(), ImmutableList.of("presto.default.abs", "presto.default.round", "presto.default.repeat"));
+
+        @Language("SQL") String queryWithAggregateFunctions = "SELECT nationkey, mktsegment, arbitrary(name), arbitrary(comment), " +
+                "approx_percentile(acctbal, 0.1), approx_percentile(acctbal, 0.3, 0.01) FROM customer GROUP BY nationkey, mktsegment";
+        resultWithQueryId = ((DistributedQueryRunner) queryRunner).executeWithQueryId(logFunctionNamesEnabledSession, queryWithAggregateFunctions);
+        queryInfo = ((DistributedQueryRunner) queryRunner).getQueryInfo(resultWithQueryId.getQueryId());
+        assertEqualsNoOrder(queryInfo.getFunctionNames(), ImmutableList.of("presto.default.arbitrary", "presto.default.approx_percentile"));
+
+        @Language("SQL") String queryWithWindowFunctions = "SELECT row_number() OVER(PARTITION BY mktsegment), nth_value(name, 5) OVER(PARTITION BY nationkey) FROM customer";
+        resultWithQueryId = ((DistributedQueryRunner) queryRunner).executeWithQueryId(logFunctionNamesEnabledSession, queryWithWindowFunctions);
+        queryInfo = ((DistributedQueryRunner) queryRunner).getQueryInfo(resultWithQueryId.getQueryId());
+        assertEqualsNoOrder(queryInfo.getFunctionNames(), ImmutableList.of("presto.default.row_number", "presto.default.nth_value"));
+
+        @Language("SQL") String queryWithNestedFunctions = "SELECT DISTINCT nationkey FROM customer WHERE mktsegment='BUILDING' AND contains(regexp_split( phone, '-' ), '11' )";
+        resultWithQueryId = ((DistributedQueryRunner) queryRunner).executeWithQueryId(logFunctionNamesEnabledSession, queryWithNestedFunctions);
+        queryInfo = ((DistributedQueryRunner) queryRunner).getQueryInfo(resultWithQueryId.getQueryId());
+        assertEqualsNoOrder(queryInfo.getFunctionNames(), ImmutableList.of("presto.default.contains", "presto.default.regexp_split"));
+    }
+
     protected String retentionDays(int days)
     {
         return "";

presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java

@@ -214,6 +214,7 @@ public final class SystemSessionProperties
     public static final String SPOOLING_OUTPUT_BUFFER_ENABLED = "spooling_output_buffer_enabled";
     public static final String SPARK_ASSIGN_BUCKET_TO_PARTITION_FOR_PARTITIONED_TABLE_WRITE_ENABLED = "spark_assign_bucket_to_partition_for_partitioned_table_write_enabled";
     public static final String LOG_FORMATTED_QUERY_ENABLED = "log_formatted_query_enabled";
+    public static final String LOG_INVOKED_FUNCTION_NAMES_ENABLED = "log_invoked_function_names_enabled";
     public static final String QUERY_RETRY_LIMIT = "query_retry_limit";
     public static final String QUERY_RETRY_MAX_EXECUTION_TIME = "query_retry_max_execution_time";
     public static final String PARTIAL_RESULTS_ENABLED = "partial_results_enabled";
@@ -1186,6 +1187,11 @@ public SystemSessionProperties(
                         "Log formatted prepared query instead of raw query when enabled",
                         featuresConfig.isLogFormattedQueryEnabled(),
                         false),
+                booleanProperty(
+                        LOG_INVOKED_FUNCTION_NAMES_ENABLED,
+                        "Log the names of the functions invoked by the query when enabled.",
+                        featuresConfig.isLogInvokedFunctionNamesEnabled(),
+                        false),
                 new PropertyMetadata<>(
                         QUERY_RETRY_LIMIT,
                         "Query retry limit due to communication failures",
@@ -2224,6 +2230,11 @@ public static boolean isLogFormattedQueryEnabled(Session session)
         return session.getSystemProperty(LOG_FORMATTED_QUERY_ENABLED, Boolean.class);
     }
 
+    public static boolean isLogInvokedFunctionNamesEnabled(Session session)
+    {
+        return session.getSystemProperty(LOG_INVOKED_FUNCTION_NAMES_ENABLED, Boolean.class);
+    }
+
     public static int getQueryRetryLimit(Session session)
     {
         return session.getSystemProperty(QUERY_RETRY_LIMIT, Integer.class);

presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java

@@ -218,6 +218,7 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur
                 ImmutableList.of(),
                 ImmutableList.of(),
                 Optional.empty(),
+                ImmutableList.of(),
                 ImmutableList.of()));
 
         logQueryTimeline(queryInfo);
@@ -251,7 +252,8 @@ public void queryCompletedEvent(QueryInfo queryInfo)
                         createPlanStatistics(queryInfo.getPlanStatsAndCosts()),
                         historyBasedPlanStatisticsTracker.getQueryStats(queryInfo).values().stream().collect(toImmutableList()),
                         queryInfo.getExpandedQuery(),
-                        queryInfo.getOptimizerInformation()));
+                        queryInfo.getOptimizerInformation(),
+                        queryInfo.getFunctionNames()));
 
         logQueryTimeline(queryInfo);
     }

presto-main/src/main/java/com/facebook/presto/execution/QueryInfo.java

@@ -92,6 +92,7 @@ public class QueryInfo
     private final Set<SqlFunctionId> removedSessionFunctions;
     private final StatsAndCosts planStatsAndCosts;
     private final List<PlanOptimizerInformation> optimizerInformation;
+    private final List<String> functionNames;
     // Using a list rather than map, to avoid implementing map key deserializer
     private final List<CanonicalPlanWithInfo> planCanonicalInfo;
 
@@ -133,6 +134,7 @@ public QueryInfo(
             @JsonProperty("removedSessionFunctions") Set<SqlFunctionId> removedSessionFunctions,
             @JsonProperty("planStatsAndCosts") StatsAndCosts planStatsAndCosts,
             @JsonProperty("optimizerInformation") List<PlanOptimizerInformation> optimizerInformation,
+            @JsonProperty("functionNames")List<String> functionNames,
             List<CanonicalPlanWithInfo> planCanonicalInfo)
     {
         requireNonNull(queryId, "queryId is null");
@@ -163,6 +165,7 @@ public QueryInfo(
         requireNonNull(removedSessionFunctions, "removedSessionFunctions is null");
         requireNonNull(planStatsAndCosts, "planStatsAndCosts is null");
         requireNonNull(optimizerInformation, "optimizerInformation is null");
+        requireNonNull(functionNames, "functionNames is null");
 
         this.queryId = queryId;
         this.session = session;
@@ -205,6 +208,7 @@ public QueryInfo(
         this.removedSessionFunctions = ImmutableSet.copyOf(removedSessionFunctions);
         this.planStatsAndCosts = planStatsAndCosts;
         this.optimizerInformation = optimizerInformation;
+        this.functionNames = functionNames;
         this.planCanonicalInfo = planCanonicalInfo == null ? ImmutableList.of() : planCanonicalInfo;
     }
 
@@ -440,6 +444,12 @@ public List<PlanOptimizerInformation> getOptimizerInformation()
         return optimizerInformation;
     }
 
+    @JsonProperty
+    public List<String> getFunctionNames()
+    {
+        return functionNames;
+    }
+
     // Don't serialize this field because it can be big
     public List<CanonicalPlanWithInfo> getPlanCanonicalInfo()
     {

presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java

@@ -161,6 +161,7 @@ public class QueryStateMachine
     private final Set<SqlFunctionId> removedSessionFunctions = Sets.newConcurrentHashSet();
 
     private final WarningCollector warningCollector;
+    private final AtomicReference<List<String>> functionNames = new AtomicReference<>(ImmutableList.of());
 
     private QueryStateMachine(
             String query,
@@ -483,6 +484,7 @@ public QueryInfo getQueryInfo(Optional<StageInfo> rootStage)
                 removedSessionFunctions,
                 Optional.ofNullable(planStatsAndCosts.get()).orElseGet(StatsAndCosts::empty),
                 session.getOptimizerInformationCollector().getOptimizationInfo(),
+                functionNames.get(),
                 Optional.ofNullable(planCanonicalInfo.get()).orElseGet(ImmutableList::of));
     }
 
@@ -550,6 +552,12 @@ public void setOutput(Optional<Output> output)
         this.output.set(output);
     }
 
+    public void setFunctionNames(List<String> functionNames)
+    {
+        requireNonNull(functionNames, "functionNames is null");
+        this.functionNames.set(ImmutableList.copyOf(functionNames));
+    }
+
     private void addSerializedCommitOutputToOutput(ConnectorCommitHandle commitHandle)
     {
         if (!output.get().isPresent()) {
@@ -1063,6 +1071,7 @@ public void pruneQueryInfo()
                 queryInfo.getRemovedSessionFunctions(),
                 StatsAndCosts.empty(),
                 queryInfo.getOptimizerInformation(),
+                queryInfo.getFunctionNames(),
                 ImmutableList.of());
         finalQueryInfo.compareAndSet(finalInfo, Optional.of(prunedQueryInfo));
     }

presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java

@@ -82,6 +82,7 @@
 
 import static com.facebook.presto.SystemSessionProperties.getExecutionPolicy;
 import static com.facebook.presto.SystemSessionProperties.getQueryAnalyzerTimeout;
+import static com.facebook.presto.SystemSessionProperties.isLogInvokedFunctionNamesEnabled;
 import static com.facebook.presto.SystemSessionProperties.isSpoolingOutputBufferEnabled;
 import static com.facebook.presto.SystemSessionProperties.isUseLegacyScheduler;
 import static com.facebook.presto.common.RuntimeMetricName.FRAGMENT_PLAN_TIME_NANOS;
@@ -228,6 +229,10 @@ private SqlQueryExecution(
 
             this.remoteTaskFactory = new TrackingRemoteTaskFactory(requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"), stateMachine);
             this.partialResultQueryManager = requireNonNull(partialResultQueryManager, "partialResultQueryManager is null");
+
+            if (isLogInvokedFunctionNamesEnabled(getSession())) {
+                stateMachine.setFunctionNames(analysis.getInvokedFunctionNames());
+            }
         }
     }
 

presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java

@@ -41,6 +41,12 @@ public Signature getSignature()
         return signature;
     }
 
+    @Override
+    public String getName()
+    {
+        return signature.getName().toString();
+    }
+
     @Override
     public CatalogSchemaName getCatalogSchemaName()
     {

presto-main/src/main/java/com/facebook/presto/metadata/SessionFunctionHandle.java

@@ -49,6 +49,12 @@ public CatalogSchemaName getCatalogSchemaName()
         return SESSION_NAMESPACE;
     }
 
+    @Override
+    public String getName()
+    {
+        return sqlFunction.getSignature().getName().toString();
+    }
+
     public FunctionMetadata getFunctionMetadata()
     {
         Signature signature = sqlFunction.getSignature();

presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java

@@ -118,6 +118,7 @@ public class FeaturesConfig
     private boolean distributedSort = true;
     private boolean optimizeJoinsWithEmptySources;
     private boolean logFormattedQueryEnabled;
+    private boolean logInvokedFunctionNamesEnabled;
 
     private boolean dictionaryAggregation;
 
@@ -1943,6 +1944,19 @@ public FeaturesConfig setLogFormattedQueryEnabled(boolean logFormattedQueryEnabl
         return this;
     }
 
+    public boolean isLogInvokedFunctionNamesEnabled()
+    {
+        return logInvokedFunctionNamesEnabled;
+    }
+
+    @Config("log-invoked-function-names-enabled")
+    @ConfigDescription("Log the names of the functions invoked by the query when enabled.")
+    public FeaturesConfig setLogInvokedFunctionNamesEnabled(boolean logInvokedFunctionNamesEnabled)
+    {
+        this.logInvokedFunctionNamesEnabled = logInvokedFunctionNamesEnabled;
+        return this;
+    }
+
     public boolean isSpoolingOutputBufferEnabled()
     {
         return spoolingOutputBufferEnabled;

presto-main/src/test/java/com/facebook/presto/execution/TestQueryInfo.java

@@ -192,6 +192,7 @@ private static QueryInfo createQueryInfo()
                 ImmutableSet.of(),
                 StatsAndCosts.empty(),
                 ImmutableList.of(),
+                ImmutableList.of(),
                 ImmutableList.of(new CanonicalPlanWithInfo(
                         new CanonicalPlan(
                                 new ValuesNode(Optional.empty(), new PlanNodeId("0"), ImmutableList.of(), ImmutableList.of(), Optional.empty()),

presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java

@@ -146,6 +146,7 @@ public void testConstructor()
                         ImmutableSet.of(),
                         StatsAndCosts.empty(),
                         ImmutableList.of(),
+                        ImmutableList.of(),
                         ImmutableList.of()));
 
         assertEquals(basicInfo.getQueryId().getId(), "0");

presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java

@@ -300,6 +300,7 @@ private QueryInfo createQueryInfo(String queryId, ResourceGroupId resourceGroupI
                 ImmutableSet.of(),
                 StatsAndCosts.empty(),
                 ImmutableList.of(),
+                ImmutableList.of(),
                 ImmutableList.of());
     }
 }

presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java

@@ -184,6 +184,7 @@ public void testDefaults()
                 .setEnforceFixedDistributionForOutputOperator(false)
                 .setEmptyJoinOptimization(false)
                 .setLogFormattedQueryEnabled(false)
+                .setLogInvokedFunctionNamesEnabled(false)
                 .setSpoolingOutputBufferEnabled(false)
                 .setSpoolingOutputBufferThreshold(new DataSize(8, MEGABYTE))
                 .setSpoolingOutputBufferTempStorage("local")
@@ -348,6 +349,7 @@ public void testExplicitPropertyMappings()
                 .put("enforce-fixed-distribution-for-output-operator", "true")
                 .put("optimizer.optimize-joins-with-empty-sources", "true")
                 .put("log-formatted-query-enabled", "true")
+                .put("log-invoked-function-names-enabled", "true")
                 .put("spooling-output-buffer-enabled", "true")
                 .put("spooling-output-buffer-threshold", "16MB")
                 .put("spooling-output-buffer-temp-storage", "tempfs")
@@ -510,6 +512,7 @@ public void testExplicitPropertyMappings()
                 .setEnforceFixedDistributionForOutputOperator(true)
                 .setEmptyJoinOptimization(true)
                 .setLogFormattedQueryEnabled(true)
+                .setLogInvokedFunctionNamesEnabled(true)
                 .setSpoolingOutputBufferEnabled(true)
                 .setSpoolingOutputBufferThreshold(new DataSize(16, MEGABYTE))
                 .setSpoolingOutputBufferTempStorage("tempfs")

presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java

@@ -38,6 +38,12 @@ public CatalogSchemaName getCatalogSchemaName()
         {
             return QualifiedObjectName.valueOf(new CatalogSchemaName("a", "b"), "c").getCatalogSchemaName();
         }
+
+        @Override
+        public String getName()
+        {
+            return "a.b.c";
+        }
     }
 
     private static final FunctionHandle TEST_FUNCTION = new TestFunctionHandle();

presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java

@@ -352,6 +352,7 @@ public static QueryInfo createQueryInfo(
                 ImmutableSet.of(),
                 planAndMore.map(PlanAndMore::getPlan).map(Plan::getStatsAndCosts).orElseGet(StatsAndCosts::empty),
                 ImmutableList.of(),
+                ImmutableList.of(),
                 planAndMore.map(PlanAndMore::getPlanCanonicalInfo).orElseGet(ImmutableList::of));
     }
 

presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryCompletedEvent.java

@@ -43,6 +43,7 @@ public class QueryCompletedEvent
     private final Instant endTime;
     private final Optional<String> expandedQuery;
     private final List<PlanOptimizerInformation> optimizerInformation;
+    private final List<String> functionNames;
 
     public QueryCompletedEvent(
             QueryMetadata metadata,
@@ -61,7 +62,8 @@ public QueryCompletedEvent(
             List<PlanStatisticsWithSourceInfo> planStatisticsRead,
             List<PlanStatisticsWithSourceInfo> planStatisticsWritten,
             Optional<String> expandedQuery,
-            List<PlanOptimizerInformation> optimizerInformation)
+            List<PlanOptimizerInformation> optimizerInformation,
+            List<String> functionNames)
     {
         this.metadata = requireNonNull(metadata, "metadata is null");
         this.statistics = requireNonNull(statistics, "statistics is null");
@@ -80,6 +82,7 @@ public QueryCompletedEvent(
         this.planStatisticsWritten = requireNonNull(planStatisticsWritten, "planStatisticsWritten is null");
         this.expandedQuery = requireNonNull(expandedQuery, "expandedQuery is null");
         this.optimizerInformation = requireNonNull(optimizerInformation, "optimizerInformation is null");
+        this.functionNames = requireNonNull(functionNames, "functionNames is null");
     }
 
     public QueryMetadata getMetadata()
@@ -166,4 +169,9 @@ public List<PlanOptimizerInformation> getOptimizerInformation()
     {
         return optimizerInformation;
     }
+
+    public List<String> getFunctionNames()
+    {
+        return functionNames;
+    }
 }

presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java

@@ -22,4 +22,6 @@
 public interface FunctionHandle
 {
     CatalogSchemaName getCatalogSchemaName();
+
+    String getName();
 }