From 137f0606757fbae44e919fac4af79b96d08dcd95 Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Fri, 16 May 2025 15:27:57 -0700 Subject: [PATCH 1/9] 1 --- .../main/protobuf/spark/connect/base.proto | 8 + .../protobuf/spark/connect/commands.proto | 2 + .../protobuf/spark/connect/pipelines.proto | 142 +++ .../connect/planner/SparkConnectPlanner.scala | 891 ++++++++++++------ 4 files changed, 756 insertions(+), 287 deletions(-) create mode 100644 sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index 1d34011b20e78..422f30e46db0f 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -26,6 +26,7 @@ import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; option java_multiple_files = true; option java_package = "org.apache.spark.connect.proto"; @@ -399,6 +400,13 @@ message ExecutePlanResponse { // ML command response MlCommandResult ml_command_result = 20; + // Response containing pipeline events that are streamed back to the client during a pipeline + // run + PipelineEventsResult pipeline_events_result = 21; + + // Pipeline command response + PipelineCommandResult pipeline_command_result = 22; + // Support arbitrary result objects. google.protobuf.Any extension = 999; } diff --git a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto index c37c98a6fdd8b..861af17e263a2 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/commands.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/commands.proto @@ -22,6 +22,7 @@ import "spark/connect/common.proto"; import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; package spark.connect; @@ -51,6 +52,7 @@ message Command { MergeIntoTableCommand merge_into_table_command = 16; MlCommand ml_command = 17; ExecuteExternalCommand execute_external_command = 18; + PipelineCommand pipeline_command = 19; // This field is used to mark extensions to the protocol. When plugins generate arbitrary // Commands they can add them here. During the planning the correct resolution is done. diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto new file mode 100644 index 0000000000000..cd65168d37a46 --- /dev/null +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -0,0 +1,142 @@ +syntax = "proto3"; + +package spark.connect; + +import "spark/connect/relations.proto"; +import "spark/connect/types.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; + +// Dispatch object for pipelines commands. +message PipelineCommand { + oneof command_type { + CreateDataflowGraph create_dataflow_graph = 1; + DefineDataset define_dataset = 2; + DefineFlow define_flow = 3; + DropDataflowGraph drop_dataflow_graph = 4; + StartRun start_run = 5; + StopRun stop_run = 6; + DefineSqlGraphElements define_sql_graph_elements = 7; + } + + message DefineSqlGraphElements { + optional string dataflow_graph_id = 1; + optional string sql_file_name = 2; + optional string sql_text = 3; + } + + // Request to create a new pipeline. + message CreateDataflowGraph { + // The default catalog. + optional string default_catalog = 1; + + // The default database. + optional string default_database = 2; + + // Default SQL configurations for all flows in this graph. + map sql_conf = 5; + + message Response { + // The ID of the created graph. + string dataflow_graph_id = 1; + } + } + + // Drops the graph and stops any running attached flows. + message DropDataflowGraph { + // The graph to drop. + string dataflow_graph_id = 1; + } + + // Request to define a dataset: a table, a materialized view, or a temporary view. + message DefineDataset { + // The graph to attach this dataset to. + string dataflow_graph_id = 1; + + // Name of the dataset. + string dataset_name = 2; + + // Table or view. + DatasetType dataset_type = 3; + + // Optional comment for the dataset. + optional string comment = 4; + + // Optional table properties. + map table_properties = 5; + + // Optional partition columns for the dataset (if applicable). + repeated string partition_cols = 6; + + // Schema for the dataset. If unset, this will be inferred. + optional spark.connect.DataType schema = 7; + + // The output table format of the dataset. + optional string format = 8; + } + + // Request to define a flow targeting a dataset. + message DefineFlow { + // The graph to attach this dataset to. + string dataflow_graph_id = 1; + + // Name of the dataset. + string flow_name = 2; + + string target_dataset_name = 3; + + // An unresolved relation that defines the dataset's flow. + spark.connect.Relation plan = 4; + + // Default SQL configurations set when running this flow. + map sql_conf = 5; + + // If true, this flow will only be run once per execution. + bool once = 6; + } + + // Resolves all datasets and flows and start a pipeline update. Should be called after all + // graph elements are registered. + message StartRun { + // The graph to start. + string dataflow_graph_id = 1; + } + message StopRun { + // The ID of the graph to stop.. + string dataflow_graph_id = 1; + } +} + +// Dispatch object for pipelines command results. +message PipelineCommandResult { + oneof result_type { + CreateDataflowGraphResult create_dataflow_graph_result = 1; + } + message CreateDataflowGraphResult { + // The ID of the created graph. + string dataflow_graph_id = 1; + } +} + +enum DatasetType { + DATASET_UNSPECIFIED = 0; + // A materialized view dataset which is published to the catalog + MATERIALIZED_VIEW = 1; + // A table which is published to the catalog + TABLE = 2; + // A view which is not published to the catalog + TEMPORARY_VIEW = 3; +} + +// A response containing events emitted during the run of a pipeline. +message PipelineEventsResult { + repeated PipelineEvent events = 1; +} + +message PipelineEvent { + // The time of the event. + optional string timestamp = 1; + // The message that should be displayed to users. + optional string message = 2; +} diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index b5a24afe4ae35..3e1b34224ace6 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -19,50 +19,166 @@ package org.apache.spark.sql.connect.planner import java.util.Properties import java.util.UUID - import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.Try import scala.util.control.NonFatal - import com.google.common.base.Throwables import com.google.common.collect.{Lists, Maps} -import com.google.protobuf.{Any => ProtoAny, ByteString} +import com.google.protobuf.{ByteString, Any => ProtoAny} import io.grpc.{Context, Status, StatusRuntimeException} import io.grpc.stub.StreamObserver import org.apache.commons.lang3.exception.ExceptionUtils - import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction} import org.apache.spark.connect.proto -import org.apache.spark.connect.proto.{CheckpointCommand, CreateResourceProfileCommand, ExecutePlanResponse, SqlCommand, StreamingForeachFunction, StreamingQueryCommand, StreamingQueryCommandResult, StreamingQueryInstanceId, StreamingQueryManagerCommand, StreamingQueryManagerCommandResult, WriteStreamOperationStart, WriteStreamOperationStartResult} +import org.apache.spark.connect.proto.{ + CheckpointCommand, + CreateResourceProfileCommand, + ExecutePlanResponse, + SqlCommand, + StreamingForeachFunction, + StreamingQueryCommand, + StreamingQueryCommandResult, + StreamingQueryInstanceId, + StreamingQueryManagerCommand, + StreamingQueryManagerCommandResult, + WriteStreamOperationStart, + WriteStreamOperationStartResult +} import org.apache.spark.connect.proto.ExecutePlanResponse.SqlCommandResult import org.apache.spark.connect.proto.Parse.ParseFormat import org.apache.spark.connect.proto.StreamingQueryManagerCommandResult.StreamingQueryInstance import org.apache.spark.connect.proto.WriteStreamOperationStart.TriggerCase -import org.apache.spark.internal.{Logging, LogKeys, MDC} +import org.apache.spark.internal.{LogKeys, Logging, MDC} import org.apache.spark.internal.LogKeys.{DATAFRAME_ID, SESSION_ID} -import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile, TaskResourceProfile, TaskResourceRequest} +import org.apache.spark.resource.{ + ExecutorResourceRequest, + ResourceProfile, + TaskResourceProfile, + TaskResourceRequest +} import org.apache.spark.sql.{Column, Encoders, ForeachWriter, Observation, Row} -import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier, QueryPlanningTracker} -import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedOrdinal, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedStarWithColumns, UnresolvedStarWithColumnsRenames, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunction, UnresolvedTranspose} -import org.apache.spark.sql.catalyst.encoders.{encoderFor, AgnosticEncoder, ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ProductEncoder, RowEncoder => AgnosticRowEncoder, StringEncoder, UnboundRowEncoder} +import org.apache.spark.sql.catalyst.{ + AliasIdentifier, + FunctionIdentifier, + QueryPlanningTracker, + expressions +} +import org.apache.spark.sql.catalyst.analysis.{ + FunctionRegistry, + GlobalTempView, + LocalTempView, + MultiAlias, + NameParameterizedQuery, + PosParameterizedQuery, + UnresolvedAlias, + UnresolvedAttribute, + UnresolvedDataFrameStar, + UnresolvedDeserializer, + UnresolvedExtractValue, + UnresolvedFunction, + UnresolvedOrdinal, + UnresolvedPlanId, + UnresolvedRegex, + UnresolvedRelation, + UnresolvedStar, + UnresolvedStarWithColumns, + UnresolvedStarWithColumnsRenames, + UnresolvedSubqueryColumnAliases, + UnresolvedTableValuedFunction, + UnresolvedTranspose +} +import org.apache.spark.sql.catalyst.encoders.{ + AgnosticEncoder, + ExpressionEncoder, + RowEncoder, + encoderFor +} +import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ + ProductEncoder, + StringEncoder, + UnboundRowEncoder, + RowEncoder => AgnosticRowEncoder +} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils} -import org.apache.spark.sql.catalyst.plans.{Cross, FullOuter, Inner, JoinType, LeftAnti, LeftOuter, LeftSemi, RightOuter, UsingJoin} +import org.apache.spark.sql.catalyst.plans.{ + Cross, + FullOuter, + Inner, + JoinType, + LeftAnti, + LeftOuter, + LeftSemi, + RightOuter, + UsingJoin +} import org.apache.spark.sql.catalyst.plans.logical -import org.apache.spark.sql.catalyst.plans.logical.{AppendColumns, Assignment, CoGroup, CollectMetrics, CommandResult, Deduplicate, DeduplicateWithinWatermark, DeleteAction, DeserializeToObject, Except, FlatMapGroupsWithState, InsertAction, InsertStarAction, Intersect, JoinWith, LocalRelation, LogicalGroupState, LogicalPlan, MapGroups, MapPartitions, MergeAction, Project, Sample, SerializeFromObject, Sort, SubqueryAlias, TimeModes, TransformWithState, TypedFilter, Union, Unpivot, UnresolvedHint, UpdateAction, UpdateEventTimeWatermarkColumn, UpdateStarAction} +import org.apache.spark.sql.catalyst.plans.logical.{ + AppendColumns, + Assignment, + CoGroup, + CollectMetrics, + CommandResult, + Deduplicate, + DeduplicateWithinWatermark, + DeleteAction, + DeserializeToObject, + Except, + FlatMapGroupsWithState, + InsertAction, + InsertStarAction, + Intersect, + JoinWith, + LocalRelation, + LogicalGroupState, + LogicalPlan, + MapGroups, + MapPartitions, + MergeAction, + Project, + Sample, + SerializeFromObject, + Sort, + SubqueryAlias, + TimeModes, + TransformWithState, + TypedFilter, + Union, + Unpivot, + UnresolvedHint, + UpdateAction, + UpdateEventTimeWatermarkColumn, + UpdateStarAction +} import org.apache.spark.sql.catalyst.streaming.InternalOutputModes import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin, TreePattern} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils} -import org.apache.spark.sql.classic.{Catalog, Dataset, MergeIntoWriter, RelationalGroupedDataset, SparkSession, TypedAggUtils, UserDefinedFunctionUtils} +import org.apache.spark.sql.classic.{ + Catalog, + Dataset, + MergeIntoWriter, + RelationalGroupedDataset, + SparkSession, + TypedAggUtils, + UserDefinedFunctionUtils +} import org.apache.spark.sql.classic.ClassicConversions._ import org.apache.spark.sql.connect.client.arrow.ArrowSerializer -import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, ForeachWriterPacket, InvalidCommandInput, InvalidPlanInput, LiteralValueProtoConverter, StorageLevelProtoConverter, StreamingListenerPacket, UdfPacket} +import org.apache.spark.sql.connect.common.{ + DataTypeProtoConverter, + ForeachWriterPacket, + InvalidCommandInput, + InvalidPlanInput, + LiteralValueProtoConverter, + StorageLevelProtoConverter, + StreamingListenerPacket, + UdfPacket +} import org.apache.spark.sql.connect.config.Connect.CONNECT_GRPC_ARROW_MAX_BATCH_SIZE import org.apache.spark.sql.connect.ml.MLHandler import org.apache.spark.sql.connect.plugin.SparkConnectPluginRegistry @@ -75,13 +191,31 @@ import org.apache.spark.sql.execution.arrow.ArrowConverters import org.apache.spark.sql.execution.command.{CreateViewCommand, ExternalCommandExecutor} import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.execution.datasources.v2.python.UserDefinedPythonDataSource -import org.apache.spark.sql.execution.python.{UserDefinedPythonFunction, UserDefinedPythonTableFunction} +import org.apache.spark.sql.execution.python.{ + UserDefinedPythonFunction, + UserDefinedPythonTableFunction +} import org.apache.spark.sql.execution.python.streaming.PythonForeachWriter import org.apache.spark.sql.execution.stat.StatFunctions import org.apache.spark.sql.execution.streaming.GroupStateImpl.groupStateTimeoutFromString import org.apache.spark.sql.execution.streaming.StreamingQueryWrapper -import org.apache.spark.sql.expressions.{Aggregator, ReduceAggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction} -import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode, StatefulProcessor, StatefulProcessorWithInitialState, StreamingQuery, StreamingQueryListener, StreamingQueryProgress, Trigger} +import org.apache.spark.sql.expressions.{ + Aggregator, + ReduceAggregator, + SparkUserDefinedFunction, + UserDefinedAggregator, + UserDefinedFunction +} +import org.apache.spark.sql.streaming.{ + GroupStateTimeout, + OutputMode, + StatefulProcessor, + StatefulProcessorWithInitialState, + StreamingQuery, + StreamingQueryListener, + StreamingQueryProgress, + Trigger +} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.{ArrowUtils, CaseInsensitiveStringMap} import org.apache.spark.storage.CacheId @@ -97,7 +231,9 @@ class SparkConnectPlanner( this(executeHolder.sessionHolder, Some(executeHolder)) } - if (!executeHolderOpt.forall { e => e.sessionHolder == sessionHolder }) { + if (!executeHolderOpt.forall { e => + e.sessionHolder == sessionHolder + }) { throw new IllegalArgumentException("executeHolder does not belong to sessionHolder") } @@ -213,8 +349,7 @@ class SparkConnectPlanner( case proto.Relation.RelTypeCase.APPLY_IN_PANDAS_WITH_STATE => transformApplyInPandasWithState(rel.getApplyInPandasWithState) case proto.Relation.RelTypeCase.COMMON_INLINE_USER_DEFINED_TABLE_FUNCTION => - transformCommonInlineUserDefinedTableFunction( - rel.getCommonInlineUserDefinedTableFunction) + transformCommonInlineUserDefinedTableFunction(rel.getCommonInlineUserDefinedTableFunction) case proto.Relation.RelTypeCase.CACHED_REMOTE_RELATION => transformCachedRemoteRelation(rel.getCachedRemoteRelation) case proto.Relation.RelTypeCase.COLLECT_METRICS => @@ -244,9 +379,9 @@ class SparkConnectPlanner( private def transformRelationPlugin(extension: ProtoAny): LogicalPlan = { SparkConnectPluginRegistry.relationRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.transform(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_.isPresent) @@ -307,7 +442,8 @@ class SparkConnectPlanner( .showString(rel.getNumRows, rel.getTruncate, rel.getVertical) LocalRelation.fromProduct( output = AttributeReference("show_string", StringType, false)() :: Nil, - data = Tuple1.apply(showString) :: Nil) + data = Tuple1.apply(showString) :: Nil + ) } private def transformHtmlString(rel: proto.HtmlString): LogicalPlan = { @@ -316,7 +452,8 @@ class SparkConnectPlanner( .htmlString(rel.getNumRows, rel.getTruncate) LocalRelation.fromProduct( output = AttributeReference("html_string", StringType, false)() :: Nil, - data = Tuple1.apply(htmlString) :: Nil) + data = Tuple1.apply(htmlString) :: Nil + ) } private def transformSql(sql: proto.SQL): LogicalPlan = { @@ -328,13 +465,15 @@ class SparkConnectPlanner( if (!namedArguments.isEmpty) { NameParameterizedQuery( parsedPlan, - namedArguments.asScala.toMap.transform((_, v) => transformExpression(v))) + namedArguments.asScala.toMap.transform((_, v) => transformExpression(v)) + ) } else if (!posArguments.isEmpty) { PosParameterizedQuery(parsedPlan, posArguments.asScala.map(transformExpression).toSeq) } else if (!args.isEmpty) { NameParameterizedQuery( parsedPlan, - args.asScala.toMap.transform((_, v) => transformLiteral(v))) + args.asScala.toMap.transform((_, v) => transformLiteral(v)) + ) } else if (!posArgs.isEmpty) { PosParameterizedQuery(parsedPlan, posArgs.asScala.map(transformLiteral).toSeq) } else { @@ -392,7 +531,8 @@ class SparkConnectPlanner( rel.getUpperBound, rel.getWithReplacement, if (rel.hasSeed) rel.getSeed else Utils.random.nextLong, - plan) + plan + ) } private def transformRepartition(rel: proto.Repartition): LogicalPlan = { @@ -418,7 +558,8 @@ class SparkConnectPlanner( if (rel.getValuesCount > 1 && rel.getValuesCount != rel.getColsCount) { throw InvalidPlanInput( s"When values contains more than 1 items, " + - s"values and cols should have the same length!") + s"values and cols should have the same length!" + ) } val dataset = Dataset.ofRows(session, transformRelation(rel.getInput)) @@ -431,8 +572,9 @@ class SparkConnectPlanner( dataset.na.fillValue(value, columns).logicalPlan } else { val valueMap = mutable.Map.empty[String, Any] - cols.zip(values).foreach { case (col, value) => - valueMap.update(col, LiteralValueProtoConverter.toCatalystValue(value)) + cols.zip(values).foreach { + case (col, value) => + valueMap.update(col, LiteralValueProtoConverter.toCatalystValue(value)) } dataset.na.fill(valueMap = valueMap.toMap).logicalPlan } @@ -460,7 +602,8 @@ class SparkConnectPlanner( rel.getReplacementsList.asScala.foreach { replace => replacement.update( LiteralValueProtoConverter.toCatalystValue(replace.getOldValue), - LiteralValueProtoConverter.toCatalystValue(replace.getNewValue)) + LiteralValueProtoConverter.toCatalystValue(replace.getNewValue) + ) } if (rel.getColsCount == 0) { @@ -522,7 +665,8 @@ class SparkConnectPlanner( LocalRelation.fromProduct( output = AttributeReference("approx_quantile", ArrayType(ArrayType(DoubleType)), false)() :: Nil, - data = Tuple1.apply(approxQuantile) :: Nil) + data = Tuple1.apply(approxQuantile) :: Nil + ) } private def transformStatCrosstab(rel: proto.StatCrosstab): LogicalPlan = { @@ -558,7 +702,10 @@ class SparkConnectPlanner( .sampleBy( col = Column(transformExpression(rel.getCol)), fractions = fractions.toMap, - seed = if (rel.hasSeed) rel.getSeed else Utils.random.nextLong) + seed = + if (rel.hasSeed) rel.getSeed + else Utils.random.nextLong + ) .logicalPlan } @@ -575,7 +722,8 @@ class SparkConnectPlanner( private def transformToDF(rel: proto.ToDF): LogicalPlan = { UnresolvedSubqueryColumnAliases( rel.getColumnNamesList.asScala.toSeq, - transformRelation(rel.getInput)) + transformRelation(rel.getInput) + ) } private def transformMapPartitions(rel: proto.MapPartitions): LogicalPlan = { @@ -601,21 +749,25 @@ class SparkConnectPlanner( DataTypeUtils.toAttributes(pythonUdf.dataType.asInstanceOf[StructType]), baseRel, isBarrier, - profile) + profile + ) case PythonEvalType.SQL_MAP_ARROW_ITER_UDF => logical.MapInArrow( pythonUdf, DataTypeUtils.toAttributes(pythonUdf.dataType.asInstanceOf[StructType]), baseRel, isBarrier, - profile) + profile + ) case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported") + s"Function with EvalType: ${pythonUdf.evalType} is not supported" + ) } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" + ) } } @@ -633,7 +785,8 @@ class SparkConnectPlanner( val mapped = MapPartitions( udf.function.asInstanceOf[Iterator[Any] => Iterator[Any]], udf.outputObjAttr, - deserialized) + deserialized + ) SerializeFromObject(udf.outputNamedExpression, mapped) } @@ -646,8 +799,8 @@ class SparkConnectPlanner( case proto.CommonInlineUserDefinedFunction.FunctionCase.PYTHON_UDF => val pythonUdf = transformPythonUDF(commonUdf) val cols = - rel.getGroupingExpressionsList.asScala.toSeq.map(expr => - Column(transformExpression(expr))) + rel.getGroupingExpressionsList.asScala.toSeq + .map(expr => Column(transformExpression(expr))) val group = Dataset .ofRows(session, transformRelation(rel.getInput)) .groupBy(cols: _*) @@ -669,12 +822,14 @@ class SparkConnectPlanner( case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported") + s"Function with EvalType: ${pythonUdf.evalType} is not supported" + ) } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" + ) } } @@ -686,7 +841,8 @@ class SparkConnectPlanner( val ds = UntypedKeyValueGroupedDataset( rel.getInput, rel.getGroupingExpressionsList, - rel.getSortingExpressionsList) + rel.getSortingExpressionsList + ) if (rel.hasTransformWithStateInfo) { val hasInitialState = !rel.getInitialGroupingExpressionsList.isEmpty && rel.hasInitialInput @@ -704,7 +860,8 @@ class SparkConnectPlanner( val initDs = UntypedKeyValueGroupedDataset( rel.getInitialInput, rel.getInitialGroupingExpressionsList, - rel.getSortingExpressionsList) + rel.getSortingExpressionsList + ) new TransformWithState( keyDeserializer, ds.valueDeserializer, @@ -720,7 +877,8 @@ class SparkConnectPlanner( initDs.groupingAttributes, initDs.dataAttributes, initDs.valueDeserializer, - initDs.analyzed) + initDs.analyzed + ) } else { val statefulProcessor = unpackedUdf.function.asInstanceOf[StatefulProcessor[Any, Any, Any]] @@ -739,7 +897,8 @@ class SparkConnectPlanner( ds.groupingAttributes, ds.dataAttributes, keyDeserializer, - LocalRelation(ds.vEncoder.schema)) + LocalRelation(ds.vEncoder.schema) + ) } val serializedPlan = SerializeFromObject(udf.outputNamedExpression, twsNode) @@ -747,7 +906,8 @@ class SparkConnectPlanner( val eventTimeWrappedPlan = UpdateEventTimeWatermarkColumn( UnresolvedAttribute(twsInfo.getEventTimeColumnName), None, - serializedPlan) + serializedPlan + ) eventTimeWrappedPlan } else { serializedPlan @@ -758,12 +918,14 @@ class SparkConnectPlanner( UntypedKeyValueGroupedDataset( rel.getInitialInput, rel.getInitialGroupingExpressionsList, - rel.getSortingExpressionsList) + rel.getSortingExpressionsList + ) } else { UntypedKeyValueGroupedDataset( rel.getInput, rel.getGroupingExpressionsList, - rel.getSortingExpressionsList) + rel.getSortingExpressionsList + ) } val timeoutConf = if (!rel.hasTimeoutConf) { GroupStateTimeout.NoTimeout @@ -779,14 +941,14 @@ class SparkConnectPlanner( val stateSchema = DataTypeProtoConverter.toCatalystType(rel.getStateSchema) match { case s: StructType => s case other => - throw InvalidPlanInput( - s"Invalid state schema dataType $other for flatMapGroupsWithState") + throw InvalidPlanInput(s"Invalid state schema dataType $other for flatMapGroupsWithState") } val stateEncoder = TypedScalaUdf.encoderFor( // the state agnostic encoder is the second element in the input encoders. unpackedUdf.inputEncoders.tail.head, "state", - Some(DataTypeUtils.toAttributes(stateSchema))) + Some(DataTypeUtils.toAttributes(stateSchema)) + ) val flatMapGroupsWithState = if (hasInitialState) { new FlatMapGroupsWithState( @@ -806,7 +968,8 @@ class SparkConnectPlanner( initialDs.dataAttributes, initialDs.valueDeserializer, initialDs.analyzed, - ds.analyzed) + ds.analyzed + ) } else { new FlatMapGroupsWithState( udf.function @@ -825,7 +988,8 @@ class SparkConnectPlanner( ds.dataAttributes, udf.inputDeserializer(ds.groupingAttributes), LocalRelation(initialDs.vEncoder.schema), // empty data set - ds.analyzed) + ds.analyzed + ) } SerializeFromObject(udf.outputNamedExpression, flatMapGroupsWithState) } else { @@ -837,7 +1001,8 @@ class SparkConnectPlanner( ds.dataAttributes, ds.sortOrder, udf.outputObjAttr, - ds.analyzed) + ds.analyzed + ) SerializeFromObject(udf.outputNamedExpression, mapped) } } @@ -850,11 +1015,11 @@ class SparkConnectPlanner( case proto.CommonInlineUserDefinedFunction.FunctionCase.PYTHON_UDF => val inputCols = - rel.getInputGroupingExpressionsList.asScala.toSeq.map(expr => - Column(transformExpression(expr))) + rel.getInputGroupingExpressionsList.asScala.toSeq + .map(expr => Column(transformExpression(expr))) val otherCols = - rel.getOtherGroupingExpressionsList.asScala.toSeq.map(expr => - Column(transformExpression(expr))) + rel.getOtherGroupingExpressionsList.asScala.toSeq + .map(expr => Column(transformExpression(expr))) val input = Dataset .ofRows(session, transformRelation(rel.getInput)) @@ -876,12 +1041,14 @@ class SparkConnectPlanner( case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported") + s"Function with EvalType: ${pythonUdf.evalType} is not supported" + ) } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" + ) } } @@ -892,11 +1059,13 @@ class SparkConnectPlanner( val left = UntypedKeyValueGroupedDataset( rel.getInput, rel.getInputGroupingExpressionsList, - rel.getInputSortingExpressionsList) + rel.getInputSortingExpressionsList + ) val right = UntypedKeyValueGroupedDataset( rel.getOther, rel.getOtherGroupingExpressionsList, - rel.getOtherSortingExpressionsList) + rel.getOtherSortingExpressionsList + ) val mapped = CoGroup( udf.function.asInstanceOf[(Any, Iterator[Any], Iterator[Any]) => IterableOnce[Any]], @@ -913,7 +1082,8 @@ class SparkConnectPlanner( right.sortOrder, udf.outputObjAttr, left.analyzed, - right.analyzed) + right.analyzed + ) SerializeFromObject(udf.outputNamedExpression, mapped) } @@ -959,7 +1129,8 @@ class SparkConnectPlanner( createFromRelationalDataset(logicalPlan, groupingExprs, sortOrder) } else { throw InvalidPlanInput( - "The grouping expression cannot be absent for KeyValueGroupedDataset") + "The grouping expression cannot be absent for KeyValueGroupedDataset" + ) } } @@ -982,7 +1153,8 @@ class SparkConnectPlanner( qe.analyzed, analyzed, aliasedGroupings, - sortOrder) + sortOrder + ) } private def createFromGroupByKeyFunc( @@ -1006,7 +1178,8 @@ class SparkConnectPlanner( withGroupingKeyAnalyzed, analyzed, withGroupingKey.newColumns, - sortOrder) + sortOrder + ) } } @@ -1062,11 +1235,17 @@ class SparkConnectPlanner( inputAttrs: Option[Seq[Attribute]] = None): ExpressionEncoder[_] = { if (encoder == UnboundRowEncoder) { inputAttrs - .map(attrs => - ExpressionEncoder(RowEncoder.encoderFor(StructType(attrs.map(a => - StructField(a.name, a.dataType, a.nullable)))))) + .map( + attrs => + ExpressionEncoder( + RowEncoder.encoderFor( + StructType(attrs.map(a => StructField(a.name, a.dataType, a.nullable))) + ) + ) + ) .getOrElse( - throw InvalidPlanInput(s"Row is not a supported $errorType type for this UDF.")) + throw InvalidPlanInput(s"Row is not a supported $errorType type for this UDF.") + ) } else { // Nested unbound row encoders val unboundTransformed = encoder match { @@ -1102,7 +1281,8 @@ class SparkConnectPlanner( outputSchema, stateSchema, rel.getOutputMode, - rel.getTimeoutConf) + rel.getTimeoutConf + ) .logicalPlan } @@ -1118,13 +1298,14 @@ class SparkConnectPlanner( case dt => throw InvalidPlanInput( "Invalid user-defined output schema type for TransformWithStateInPandas. " + - s"Expect a struct type, but got ${dt.typeName}.") + s"Expect a struct type, but got ${dt.typeName}." + ) } } if (rel.hasInitialInput) { - val initialGroupingCols = rel.getInitialGroupingExpressionsList.asScala.toSeq.map(expr => - Column(transformExpression(expr))) + val initialGroupingCols = rel.getInitialGroupingExpressionsList.asScala.toSeq + .map(expr => Column(transformExpression(expr))) val initialStateDs = Dataset .ofRows(session, transformRelation(rel.getInitialInput)) @@ -1144,7 +1325,8 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, initialStateDs, - twsInfo.getEventTimeColumnName) + twsInfo.getEventTimeColumnName + ) .logicalPlan } else { // use Row @@ -1155,7 +1337,8 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, initialStateDs, - twsInfo.getEventTimeColumnName) + twsInfo.getEventTimeColumnName + ) .logicalPlan } @@ -1168,7 +1351,8 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, null, - twsInfo.getEventTimeColumnName) + twsInfo.getEventTimeColumnName + ) .logicalPlan } else { // use Row @@ -1179,7 +1363,8 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, null, - twsInfo.getEventTimeColumnName) + twsInfo.getEventTimeColumnName + ) .logicalPlan } } @@ -1192,10 +1377,12 @@ class SparkConnectPlanner( val function = createPythonUserDefinedTableFunction(fun) function.builder( fun.getArgumentsList.asScala.map(transformExpression).toSeq, - session.sessionState.sqlParser) + session.sessionState.sqlParser + ) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" + ) } } @@ -1210,7 +1397,8 @@ class SparkConnectPlanner( // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull) + accumulator = sessionHolder.pythonAccumulator.orNull + ) } private def transformPythonDataSource(ds: proto.PythonDataSource): SimplePythonFunction = { @@ -1224,7 +1412,8 @@ class SparkConnectPlanner( // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull) + accumulator = sessionHolder.pythonAccumulator.orNull + ) } private def transformCachedRemoteRelation(rel: proto.CachedRemoteRelation): LogicalPlan = { @@ -1244,7 +1433,8 @@ class SparkConnectPlanner( } Project( Seq(UnresolvedStarWithColumnsRenames(existingNames = colNames, newNames = newColNames)), - transformRelation(rel.getInput)) + transformRelation(rel.getInput) + ) } private def transformWithColumns(rel: proto.WithColumns): LogicalPlan = { @@ -1269,8 +1459,11 @@ class SparkConnectPlanner( UnresolvedStarWithColumns( colNames = colNames, exprs = exprs, - explicitMetadata = Some(metadata))), - transformRelation(rel.getInput)) + explicitMetadata = Some(metadata) + ) + ), + transformRelation(rel.getInput) + ) } private def transformWithWatermark(rel: proto.WithWatermark): LogicalPlan = { @@ -1300,7 +1493,8 @@ class SparkConnectPlanner( .getOrElse { throw InvalidPlanInput( s"Not found any cached local relation with the hash: ${blockId.hash} in " + - s"the session with sessionUUID ${blockId.sessionUUID}.") + s"the session with sessionUUID ${blockId.sessionUUID}." + ) } } @@ -1321,7 +1515,8 @@ class SparkConnectPlanner( rel: proto.UnresolvedTableValuedFunction): LogicalPlan = { UnresolvedTableValuedFunction( rel.getFunctionName, - rel.getArgumentsList.asScala.map(transformExpression).toSeq) + rel.getArgumentsList.asScala.map(transformExpression).toSeq + ) } private def transformUnpivot(rel: proto.Unpivot): LogicalPlan = { @@ -1336,7 +1531,8 @@ class SparkConnectPlanner( None, rel.getVariableColumnName, Seq(rel.getValueColumnName), - transformRelation(rel.getInput)) + transformRelation(rel.getInput) + ) } else { val values = rel.getValues.getValuesList.asScala.toArray.map { expr => Column(transformExpression(expr)) @@ -1348,12 +1544,12 @@ class SparkConnectPlanner( None, rel.getVariableColumnName, Seq(rel.getValueColumnName), - transformRelation(rel.getInput)) + transformRelation(rel.getInput) + ) } } - private def transformRepartitionByExpression( - rel: proto.RepartitionByExpression): LogicalPlan = { + private def transformRepartitionByExpression(rel: proto.RepartitionByExpression): LogicalPlan = { val numPartitionsOpt = if (rel.hasNumPartitions) { Some(rel.getNumPartitions) } else { @@ -1363,7 +1559,8 @@ class SparkConnectPlanner( logical.RepartitionByExpression( partitionExpressions, transformRelation(rel.getInput), - numPartitionsOpt) + numPartitionsOpt + ) } private def transformCollectMetrics(rel: proto.CollectMetrics, planId: Long): LogicalPlan = { @@ -1394,7 +1591,8 @@ class SparkConnectPlanner( } if (!rel.getAllColumnsAsKeys && rel.getColumnNamesCount == 0) { throw InvalidPlanInput( - "Deduplicate requires to either deduplicate on all columns or a subset of columns") + "Deduplicate requires to either deduplicate on all columns or a subset of columns" + ) } val queryExecution = new QueryExecution(session, transformRelation(rel.getInput)) val resolver = session.sessionState.analyzer.resolver @@ -1451,7 +1649,8 @@ class SparkConnectPlanner( val schemaType = DataType.parseTypeWithFallback( rel.getSchema, parseDatatypeString, - fallbackParser = DataType.fromJson) + fallbackParser = DataType.fromJson + ) schema = schemaType match { case s: StructType => s case d => StructType(Seq(StructField("value", d))) @@ -1461,7 +1660,8 @@ class SparkConnectPlanner( if (rel.hasData) { val (rows, structType) = ArrowConverters.fromBatchWithSchemaIterator( Iterator(rel.getData.toByteArray), - TaskContext.get()) + TaskContext.get() + ) if (structType == null) { throw InvalidPlanInput(s"Input data for LocalRelation does not produce a schema.") } @@ -1493,7 +1693,8 @@ class SparkConnectPlanner( val project = Dataset .ofRows( session, - logicalPlan = logical.LocalRelation(normalize(structType).asInstanceOf[StructType])) + logicalPlan = logical.LocalRelation(normalize(structType).asInstanceOf[StructType]) + ) .toDF(normalized.names.toImmutableArraySeq: _*) .to(normalized) .logicalPlan @@ -1502,12 +1703,14 @@ class SparkConnectPlanner( val proj = UnsafeProjection.create(project.projectList, project.child.output) logical.LocalRelation( DataTypeUtils.toAttributes(schema), - data.map(proj).map(_.copy()).toSeq) + data.map(proj).map(_.copy()).toSeq + ) } } else { if (schema == null) { throw InvalidPlanInput( - s"Schema for LocalRelation is required when the input data is not provided.") + s"Schema for LocalRelation is required when the input data is not provided." + ) } LocalRelation(schema) } @@ -1515,10 +1718,7 @@ class SparkConnectPlanner( /** Parse as DDL, with a fallback to JSON. Throws an exception if if fails to parse. */ private def parseSchema(schema: String): StructType = { - DataType.parseTypeWithFallback( - schema, - StructType.fromDDL, - fallbackParser = DataType.fromJson) match { + DataType.parseTypeWithFallback(schema, StructType.fromDDL, fallbackParser = DataType.fromJson) match { case s: StructType => s case other => throw InvalidPlanInput(s"Invalid schema $other") } @@ -1531,7 +1731,8 @@ class SparkConnectPlanner( UnresolvedRelation( parser.parseMultipartIdentifier(rel.getNamedTable.getUnparsedIdentifier), new CaseInsensitiveStringMap(rel.getNamedTable.getOptionsMap), - isStreaming = rel.getIsStreaming) + isStreaming = rel.getIsStreaming + ) case proto.Read.ReadTypeCase.DATA_SOURCE if !rel.getIsStreaming => val reader = session.read @@ -1565,7 +1766,8 @@ class SparkConnectPlanner( } } else { throw InvalidPlanInput( - s"Predicates are not supported for ${rel.getDataSource.getFormat} data sources.") + s"Predicates are not supported for ${rel.getDataSource.getFormat} data sources." + ) } case proto.Read.ReadTypeCase.DATA_SOURCE if rel.getIsStreaming => @@ -1632,9 +1834,9 @@ class SparkConnectPlanner( val udf = expr.getCommonInlineUserDefinedFunction // A typed scala udf is a scala udf && the udf argument is an unresolved start. udf.getFunctionCase == - proto.CommonInlineUserDefinedFunction.FunctionCase.SCALAR_SCALA_UDF && - udf.getArgumentsCount == 1 && - udf.getArguments(0).getExprTypeCase == proto.Expression.ExprTypeCase.UNRESOLVED_STAR + proto.CommonInlineUserDefinedFunction.FunctionCase.SCALAR_SCALA_UDF && + udf.getArgumentsCount == 1 && + udf.getArguments(0).getExprTypeCase == proto.Expression.ExprTypeCase.UNRESOLVED_STAR case _ => false } @@ -1656,8 +1858,8 @@ class SparkConnectPlanner( } val logicalPlan = - if (rel.getExpressionsList.asScala.toSeq.exists( - _.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { + if (rel.getExpressionsList.asScala.toSeq + .exists(_.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { session.sessionState.executePlan(baseRel).analyzed } else { baseRel @@ -1736,7 +1938,8 @@ class SparkConnectPlanner( Some(origin.getStackTraceList.asScala.map(transformStackTraceElement).toArray) } else { None - }) + } + ) } private def transformStackTraceElement(element: proto.StackTraceElement): StackTraceElement = { @@ -1747,7 +1950,8 @@ class SparkConnectPlanner( /* @NotNull declaringClass */ element.getDeclaringClass, /* @NotNull methodName */ element.getMethodName, /* fileName */ if (element.hasFileName) element.getFileName else null, - /* lineNumber */ element.getLineNumber) + /* lineNumber */ element.getLineNumber + ) } private def doTransformExpression( @@ -1794,7 +1998,8 @@ class SparkConnectPlanner( transformSubqueryExpression(exp.getSubqueryExpression) case _ => throw InvalidPlanInput( - s"Expression with ID: ${exp.getExprTypeCase.getNumber} is not supported") + s"Expression with ID: ${exp.getExprTypeCase.getNumber} is not supported" + ) } } @@ -1817,9 +2022,9 @@ class SparkConnectPlanner( private def transformExpressionPlugin(extension: ProtoAny): Expression = { SparkConnectPluginRegistry.expressionRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.transform(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_.isPresent) @@ -1839,31 +2044,34 @@ class SparkConnectPlanner( private def transformLimit(limit: proto.Limit): LogicalPlan = { logical.Limit( limitExpr = expressions.Literal(limit.getLimit, IntegerType), - transformRelation(limit.getInput)) + transformRelation(limit.getInput) + ) } private def transformTail(tail: proto.Tail): LogicalPlan = { logical.Tail( limitExpr = expressions.Literal(tail.getLimit, IntegerType), - transformRelation(tail.getInput)) + transformRelation(tail.getInput) + ) } private def transformOffset(offset: proto.Offset): LogicalPlan = { logical.Offset( offsetExpr = expressions.Literal(offset.getOffset, IntegerType), - transformRelation(offset.getInput)) + transformRelation(offset.getInput) + ) } /** * Translates a scalar function from proto to the Catalyst expression. */ - private def transformUnresolvedFunction( - fun: proto.Expression.UnresolvedFunction): Expression = { + private def transformUnresolvedFunction(fun: proto.Expression.UnresolvedFunction): Expression = { if (fun.getIsUserDefinedFunction) { UnresolvedFunction( parser.parseMultipartIdentifier(fun.getFunctionName), fun.getArgumentsList.asScala.map(transformExpression).toSeq, - isDistinct = fun.getIsDistinct) + isDistinct = fun.getIsDistinct + ) } else { val name = fun.getFunctionName val internal = if (fun.hasIsInternal) { @@ -1881,7 +2089,8 @@ class SparkConnectPlanner( name :: Nil, fun.getArgumentsList.asScala.map(transformExpression).toSeq, isDistinct = fun.getIsDistinct, - isInternal = internal) + isInternal = internal + ) } } @@ -1902,7 +2111,8 @@ class SparkConnectPlanner( transformScalaUDF(fun) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" + ) } } @@ -1919,7 +2129,8 @@ class SparkConnectPlanner( UnresolvedFunction( parser.parseMultipartIdentifier(funcName), fun.getArgumentsList.asScala.map(transformExpression).toSeq, - false) + false + ) } private def transformNamedArgumentExpression( @@ -1945,13 +2156,15 @@ class SparkConnectPlanner( case nsm: NoSuchMethodException => throw new ClassNotFoundException( s"Failed to load class correctly due to $nsm. " + - "Make sure the artifact where the class is defined is installed by calling" + - " session.addArtifact.") + "Make sure the artifact where the class is defined is installed by calling" + + " session.addArtifact." + ) case cnf: ClassNotFoundException => throw new ClassNotFoundException( s"Failed to load class: ${cnf.getMessage}. " + - "Make sure the artifact where the class is defined is installed by calling" + - " session.addArtifact.") + "Make sure the artifact where the class is defined is installed by calling" + + " session.addArtifact." + ) case _ => throw t } } @@ -1974,8 +2187,7 @@ class SparkConnectPlanner( case uda: UserDefinedAggregator[_, _, _] => ScalaAggregator(uda, children).toAggregateExpression(fun.getIsDistinct) case other => - throw InvalidPlanInput( - s"Unsupported UserDefinedFunction implementation: ${other.getClass}") + throw InvalidPlanInput(s"Unsupported UserDefinedFunction implementation: ${other.getClass}") } } @@ -1999,7 +2211,8 @@ class SparkConnectPlanner( inputEncoder = ExpressionEncoder(udfPacket.inputEncoders.head), givenName = Option(fun.getFunctionName), nullable = udf.getNullable, - deterministic = fun.getDeterministic) + deterministic = fun.getDeterministic + ) } else { SparkUserDefinedFunction( f = udfPacket.function, @@ -2008,7 +2221,8 @@ class SparkConnectPlanner( outputEncoder = Option(ExpressionEncoder(udfPacket.outputEncoder)), givenName = Option(fun.getFunctionName), nullable = udf.getNullable, - deterministic = fun.getDeterministic) + deterministic = fun.getDeterministic + ) } } @@ -2042,7 +2256,8 @@ class SparkConnectPlanner( func = function, dataType = transformDataType(udf.getOutputType), pythonEvalType = udf.getEvalType, - udfDeterministic = fun.getDeterministic) + udfDeterministic = fun.getDeterministic + ) } private def transformPythonFunction(fun: proto.PythonUDF): SimplePythonFunction = { @@ -2053,12 +2268,13 @@ class SparkConnectPlanner( pythonExec = pythonExec, // Merge the user specified includes with the includes managed by the artifact manager. pythonIncludes = (fun.getAdditionalIncludesList.asScala.toSeq ++ - sessionHolder.artifactManager.getPythonIncludes).asJava, + sessionHolder.artifactManager.getPythonIncludes).asJava, pythonVer = fun.getPythonVer, // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull) + accumulator = sessionHolder.pythonAccumulator.orNull + ) } /** @@ -2068,13 +2284,15 @@ class SparkConnectPlanner( if (lambda.getArgumentsCount == 0 || lambda.getArgumentsCount > 3) { throw InvalidPlanInput( "LambdaFunction requires 1 ~ 3 arguments, " + - s"but got ${lambda.getArgumentsCount} ones!") + s"but got ${lambda.getArgumentsCount} ones!" + ) } LambdaFunction( function = transformExpression(lambda.getFunction), arguments = lambda.getArgumentsList.asScala.toSeq - .map(transformUnresolvedNamedLambdaVariable)) + .map(transformUnresolvedNamedLambdaVariable) + ) } private def transformUnresolvedNamedLambdaVariable( @@ -2097,7 +2315,8 @@ class SparkConnectPlanner( } else { if (alias.hasMetadata) { throw InvalidPlanInput( - "Alias expressions with more than 1 identifier must not use optional metadata.") + "Alias expressions with more than 1 identifier must not use optional metadata." + ) } MultiAlias(transformExpression(alias.getExpr), alias.getNameList.asScala.toSeq) } @@ -2118,7 +2337,8 @@ class SparkConnectPlanner( val target = star.getUnparsedTarget if (!target.endsWith(".*")) { throw InvalidPlanInput( - s"UnresolvedStar requires a unparsed target ending with '.*', but got $target.") + s"UnresolvedStar requires a unparsed target ending with '.*', but got $target." + ) } val parts = UnresolvedAttribute.parseAttributeName(target.dropRight(2)) UnresolvedStar(Some(parts)) @@ -2172,7 +2392,8 @@ class SparkConnectPlanner( extract: proto.Expression.UnresolvedExtractValue): UnresolvedExtractValue = { UnresolvedExtractValue( transformExpression(extract.getChild), - transformExpression(extract.getExtraction)) + transformExpression(extract.getExtraction) + ) } private def transformUpdateFields(update: proto.Expression.UpdateFields): UpdateFields = { @@ -2181,12 +2402,14 @@ class SparkConnectPlanner( UpdateFields.apply( col = transformExpression(update.getStructExpression), fieldName = update.getFieldName, - expr = transformExpression(update.getValueExpression)) + expr = transformExpression(update.getValueExpression) + ) } else { // drop a field UpdateFields.apply( col = transformExpression(update.getStructExpression), - fieldName = update.getFieldName) + fieldName = update.getFieldName + ) } } @@ -2247,11 +2470,13 @@ class SparkConnectPlanner( val windowSpec = WindowSpecDefinition( partitionSpec = window.getPartitionSpecList.asScala.toSeq.map(transformExpression), orderSpec = window.getOrderSpecList.asScala.toSeq.map(transformSortOrder), - frameSpecification = frameSpec) + frameSpecification = frameSpec + ) WindowExpression( windowFunction = transformExpression(window.getWindowFunction), - windowSpec = windowSpec) + windowSpec = windowSpec + ) } private def transformSetOperation(u: proto.SetOperation): LogicalPlan = { @@ -2276,7 +2501,8 @@ class SparkConnectPlanner( case proto.SetOperation.SetOpType.SET_OP_TYPE_UNION => if (!u.getByName && u.getAllowMissingColumns) { throw InvalidPlanInput( - "UnionByName `allowMissingCol` can be true only if `byName` is true.") + "UnionByName `allowMissingCol` can be true only if `byName` is true." + ) } val union = Union(Seq(leftPlan, rightPlan), u.getByName, u.getAllowMissingColumns) if (isAll) { @@ -2299,7 +2525,8 @@ class SparkConnectPlanner( session.sessionState.conf.dataFrameSelfJoinAutoResolveAmbiguity, session.sessionState.analyzer.resolver, rel.getJoinDataType.getIsLeftStruct, - rel.getJoinDataType.getIsRightStruct) + rel.getJoinDataType.getIsRightStruct + ) } private def transformJoinOrJoinWith(rel: proto.Join): LogicalPlan = { @@ -2314,12 +2541,14 @@ class SparkConnectPlanner( assertPlan(rel.hasLeft && rel.hasRight, "Both join sides must be present") if (rel.hasJoinCondition && rel.getUsingColumnsCount > 0) { throw InvalidPlanInput( - s"Using columns or join conditions cannot be set at the same time in Join") + s"Using columns or join conditions cannot be set at the same time in Join" + ) } val joinCondition = if (rel.hasJoinCondition) Some(transformExpression(rel.getJoinCondition)) else None val catalystJointype = transformJoinType( - if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER) + if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER + ) val joinType = if (rel.getUsingColumnsCount > 0) { UsingJoin(catalystJointype, rel.getUsingColumnsList.asScala.toSeq) } else { @@ -2330,7 +2559,8 @@ class SparkConnectPlanner( right = transformRelation(rel.getRight), joinType = joinType, condition = joinCondition, - hint = logical.JoinHint.NONE) + hint = logical.JoinHint.NONE + ) } private def transformJoinType(t: proto.Join.JoinType): JoinType = { @@ -2366,7 +2596,8 @@ class SparkConnectPlanner( joinType = joinType, tolerance = tolerance, allowExactMatches = allowExactMatches, - direction = direction) + direction = direction + ) } else { val joinExprs = if (rel.hasJoinExpr) Column(transformExpression(rel.getJoinExpr)) else null left.joinAsOf( @@ -2377,7 +2608,8 @@ class SparkConnectPlanner( joinType = joinType, tolerance = tolerance, allowExactMatches = allowExactMatches, - direction = direction) + direction = direction + ) } joined.logicalPlan } @@ -2387,12 +2619,14 @@ class SparkConnectPlanner( val joinCondition = if (rel.hasJoinCondition) Some(transformExpression(rel.getJoinCondition)) else None val joinType = transformJoinType( - if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER) + if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER + ) logical.LateralJoin( left = transformRelation(rel.getLeft), right = LateralSubquery(transformRelation(rel.getRight)), joinType = joinType, - condition = joinCondition) + condition = joinCondition + ) } private def transformSort(sort: proto.Sort): LogicalPlan = { @@ -2400,7 +2634,8 @@ class SparkConnectPlanner( logical.Sort( child = transformRelation(sort.getInput), global = sort.getIsGlobal, - order = sort.getOrderList.asScala.toSeq.map(transformSortOrder)) + order = sort.getOrderList.asScala.toSeq.map(transformSortOrder) + ) } private def transformSortOrder(order: proto.Expression.SortOrder) = { @@ -2416,7 +2651,8 @@ class SparkConnectPlanner( expressions.NullsFirst case _ => expressions.NullsLast }, - sameOrderExpressions = Seq.empty) + sameOrderExpressions = Seq.empty + ) } /** @@ -2452,7 +2688,7 @@ class SparkConnectPlanner( // This is the case for datasets created via groupByKey, // and also via RelationalGroupedDS#as, as the first is a dummy UDF currently. if rel.getGroupingExpressionsList.size() >= 1 && - isTypedScalaUdfExpr(rel.getGroupingExpressionsList.get(0)) => + isTypedScalaUdfExpr(rel.getGroupingExpressionsList.get(0)) => transformKeyValueGroupedAggregate(rel) case _ => transformRelationalGroupedAggregate(rel) @@ -2477,8 +2713,8 @@ class SparkConnectPlanner( val input = transformRelation(rel.getInput) val logicalPlan = - if (rel.getAggregateExpressionsList.asScala.toSeq.exists( - _.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { + if (rel.getAggregateExpressionsList.asScala.toSeq + .exists(_.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { session.sessionState.executePlan(input).analyzed } else { input @@ -2495,19 +2731,22 @@ class SparkConnectPlanner( logical.Aggregate( groupingExpressions = groupingExpressionsWithOrdinals, aggregateExpressions = aliasedAgg, - child = logicalPlan) + child = logicalPlan + ) case proto.Aggregate.GroupType.GROUP_TYPE_ROLLUP => logical.Aggregate( groupingExpressions = Seq(Rollup(groupingExpressionsWithOrdinals.map(Seq(_)))), aggregateExpressions = aliasedAgg, - child = logicalPlan) + child = logicalPlan + ) case proto.Aggregate.GroupType.GROUP_TYPE_CUBE => logical.Aggregate( groupingExpressions = Seq(Cube(groupingExpressionsWithOrdinals.map(Seq(_)))), aggregateExpressions = aliasedAgg, - child = logicalPlan) + child = logicalPlan + ) case proto.Aggregate.GroupType.GROUP_TYPE_PIVOT => if (!rel.hasPivot) { @@ -2527,7 +2766,8 @@ class SparkConnectPlanner( pivotColumn = pivotExpr, pivotValues = valueExprs, aggregates = aggExprs, - child = logicalPlan) + child = logicalPlan + ) case proto.Aggregate.GroupType.GROUP_TYPE_GROUPING_SETS => val groupingSetsExpressionsWithOrdinals = @@ -2539,9 +2779,12 @@ class SparkConnectPlanner( groupingExpressions = Seq( GroupingSets( groupingSets = groupingSetsExpressionsWithOrdinals, - userGivenGroupByExprs = groupingExpressionsWithOrdinals)), + userGivenGroupByExprs = groupingExpressionsWithOrdinals + ) + ), aggregateExpressions = aliasedAgg, - child = logicalPlan) + child = logicalPlan + ) case other => throw InvalidPlanInput(s"Unknown Group Type $other") } @@ -2570,7 +2813,7 @@ class SparkConnectPlanner( val udf = fun.getArgumentsList.asScala match { case collection.Seq(e) if e.hasCommonInlineUserDefinedFunction && - e.getCommonInlineUserDefinedFunction.hasScalarScalaUdf => + e.getCommonInlineUserDefinedFunction.hasScalarScalaUdf => unpackUdf(e.getCommonInlineUserDefinedFunction) case other => throw InvalidPlanInput(s"reduce should carry a scalar scala udf, but got $other") @@ -2607,10 +2850,8 @@ class SparkConnectPlanner( TypedAggregateExpression(aggregator)(aggregator.bufferEncoder, aggregator.outputEncoder) val taeWithInput = baseRelationOpt match { case Some(baseRelation) => - val inputEncoder = TypedScalaUdf.encoderFor( - udfPacket.inputEncoders.head, - "input", - Some(baseRelation.output)) + val inputEncoder = + TypedScalaUdf.encoderFor(udfPacket.inputEncoders.head, "input", Some(baseRelation.output)) TypedAggUtils .withInputType(tae, inputEncoder, baseRelation.output) .asInstanceOf[TypedAggregateExpression] @@ -2686,18 +2927,18 @@ class SparkConnectPlanner( case proto.Command.CommandTypeCase.STREAMING_QUERY_MANAGER_COMMAND => handleStreamingQueryManagerCommand( command.getStreamingQueryManagerCommand, - responseObserver) + responseObserver + ) case proto.Command.CommandTypeCase.STREAMING_QUERY_LISTENER_BUS_COMMAND => val handler = new SparkConnectStreamingQueryListenerHandler(executeHolder) - handler.handleListenerCommand( - command.getStreamingQueryListenerBusCommand, - responseObserver) + handler.handleListenerCommand(command.getStreamingQueryListenerBusCommand, responseObserver) case proto.Command.CommandTypeCase.GET_RESOURCES_COMMAND => handleGetResourcesCommand(responseObserver) case proto.Command.CommandTypeCase.CREATE_RESOURCE_PROFILE_COMMAND => handleCreateResourceProfileCommand( command.getCreateResourceProfileCommand, - responseObserver) + responseObserver + ) case proto.Command.CommandTypeCase.CHECKPOINT_COMMAND => handleCheckpointCommand(command.getCheckpointCommand, responseObserver) case proto.Command.CommandTypeCase.REMOVE_CACHED_REMOTE_RELATION_COMMAND => @@ -2706,6 +2947,8 @@ class SparkConnectPlanner( handleMergeIntoTableCommand(command.getMergeIntoTableCommand) case proto.Command.CommandTypeCase.ML_COMMAND => handleMlCommand(command.getMlCommand, responseObserver) + case proto.Command.CommandTypeCase.PIPELINE_COMMAND => + handlePipelineCommand(command.getPipelineCommand, responseObserver) case proto.Command.CommandTypeCase.EXECUTE_EXTERNAL_COMMAND => handleExecuteExternalCommand(command.getExecuteExternalCommand, responseObserver) @@ -2724,7 +2967,14 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setMlCommandResult(result) - .build()) + .build() + ) + } + + private def handlePipelineCommand( + command: proto.PipelineCommand, + responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { + throw InvalidPlanInput(s"PipelineCommand is not supported in Spark Connect.") } private def handleExecuteExternalCommand( @@ -2740,7 +2990,8 @@ class SparkConnectPlanner( session, command.getRunner, command.getCommand, - command.getOptionsMap.asScala.toMap) + command.getOptionsMap.asScala.toMap + ) val result = executor.execute() executeHolder.eventsManager.postFinished(Some(result.size)) @@ -2750,7 +3001,8 @@ class SparkConnectPlanner( StringEncoder, ArrowUtils.rootAllocator, session.sessionState.conf.sessionLocalTimeZone, - session.sessionState.conf.arrowUseLargeVarTypes) + session.sessionState.conf.arrowUseLargeVarTypes + ) val sqlCommandResult = SqlCommandResult.newBuilder() sqlCommandResult.getRelationBuilder.getLocalRelationBuilder.setData(arrowData) responseObserver.onNext( @@ -2759,7 +3011,8 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setSqlCommandResult(sqlCommandResult) - .build()) + .build() + ) } private def handleSqlCommand( @@ -2781,7 +3034,8 @@ class SparkConnectPlanner( .putAllNamedArguments(command.getNamedArgumentsMap) .addAllPosArgs(command.getPosArgsList) .addAllPosArguments(command.getPosArgumentsList) - .build()) + .build() + ) .build() } @@ -2792,7 +3046,8 @@ class SparkConnectPlanner( executeSQLWithRefs(relation.getWithRelations, tracker) case other => throw InvalidPlanInput( - s"SQL command expects either a SQL or a WithRelations, but got $other") + s"SQL command expects either a SQL or a WithRelations, but got $other" + ) } // Check if commands have been executed. @@ -2821,7 +3076,8 @@ class SparkConnectPlanner( schema, timeZoneId, errorOnDuplicatedFieldNames = false, - largeVarTypes = largeVarTypes) + largeVarTypes = largeVarTypes + ) } else { val batches = ArrowConverters.toBatchWithSchemaIterator( rowIter = rows.iterator, @@ -2830,7 +3086,8 @@ class SparkConnectPlanner( maxEstimatedBatchSize = maxBatchSize, timeZoneId = timeZoneId, errorOnDuplicatedFieldNames = false, - largeVarTypes = largeVarTypes) + largeVarTypes = largeVarTypes + ) assert(batches.hasNext) val bytes = batches.next() assert(!batches.hasNext, s"remaining batches: ${batches.size}") @@ -2843,7 +3100,9 @@ class SparkConnectPlanner( .setLocalRelation( proto.LocalRelation .newBuilder() - .setData(ByteString.copyFrom(bytes)))) + .setData(ByteString.copyFrom(bytes)) + ) + ) metrics.addAllMetrics(MetricGenerator.transformPlan(df).asJava) } else { // No execution triggered for relations. Manually set ready @@ -2858,7 +3117,8 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setSqlCommandResult(result) - .build()) + .build() + ) // Send Metrics when isCommand (i.e. show tables) which is eagerly executed & has metrics // Skip metrics when !isCommand (i.e. select 1) which is not executed & doesn't have metrics @@ -2869,7 +3129,8 @@ class SparkConnectPlanner( .setSessionId(sessionHolder.sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setMetrics(metrics.build) - .build) + .build + ) } } @@ -2928,17 +3189,20 @@ class SparkConnectPlanner( session.sql( sql.getQuery, namedArguments.asScala.toMap.transform((_, e) => Column(transformExpression(e))), - tracker) + tracker + ) } else if (!posArguments.isEmpty) { session.sql( sql.getQuery, posArguments.asScala.map(e => Column(transformExpression(e))).toArray, - tracker) + tracker + ) } else if (!args.isEmpty) { session.sql( sql.getQuery, args.asScala.toMap.transform((_, v) => transformLiteral(v)), - tracker) + tracker + ) } else if (!posArgs.isEmpty) { session.sql(sql.getQuery, posArgs.asScala.map(transformLiteral).toArray, tracker) } else { @@ -2957,7 +3221,8 @@ class SparkConnectPlanner( handleRegisterScalaUDF(fun) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" + ) } executeHolder.eventsManager.postFinished() } @@ -2970,7 +3235,8 @@ class SparkConnectPlanner( session.udtf.registerPython(fun.getFunctionName, function) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" + ) } executeHolder.eventsManager.postFinished() } @@ -2984,7 +3250,8 @@ class SparkConnectPlanner( session.dataSource.registerPython(fun.getName, dataSource) case _ => throw InvalidPlanInput( - s"Data source with ID: ${fun.getDataSourceCase.getNumber} is not supported") + s"Data source with ID: ${fun.getDataSourceCase.getNumber} is not supported" + ) } executeHolder.eventsManager.postFinished() } @@ -2998,7 +3265,8 @@ class SparkConnectPlanner( case dt => throw InvalidPlanInput( "Invalid Python user-defined table function return type. " + - s"Expect a struct type, but got ${dt.typeName}.") + s"Expect a struct type, but got ${dt.typeName}." + ) } } else { None @@ -3009,7 +3277,8 @@ class SparkConnectPlanner( func = transformPythonTableFunction(udtf), returnType = returnType, pythonEvalType = udtf.getEvalType, - udfDeterministic = fun.getDeterministic) + udfDeterministic = fun.getDeterministic + ) } private def handleRegisterPythonUDF(fun: proto.CommonInlineUserDefinedFunction): Unit = { @@ -3038,9 +3307,9 @@ class SparkConnectPlanner( private def handleCommandPlugin(extension: ProtoAny): Unit = { SparkConnectPluginRegistry.commandRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.process(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_ == true) @@ -3069,7 +3338,8 @@ class SparkConnectPlanner( plan = transformRelation(createView.getInput), allowExisting = false, replace = createView.getReplace, - viewType = viewType) + viewType = viewType + ) val tracker = executeHolder.eventsManager.createQueryPlanningTracker() Dataset.ofRows(session, plan, tracker).queryExecution.commandExecuted @@ -3111,7 +3381,8 @@ class SparkConnectPlanner( val cols = op.getBucketColumnNamesList.asScala if (op.getNumBuckets <= 0) { throw InvalidCommandInput( - s"BucketBy must specify a bucket count > 0, received ${op.getNumBuckets} instead.") + s"BucketBy must specify a bucket count > 0, received ${op.getNumBuckets} instead." + ) } w.bucketBy(op.getNumBuckets, cols.head, cols.tail.toSeq: _*) } @@ -3143,12 +3414,14 @@ class SparkConnectPlanner( case _ => throw new UnsupportedOperationException( "WriteOperation:SaveTable:TableSaveMethod not supported " - + s"${writeOperation.getTable.getSaveMethodValue}") + + s"${writeOperation.getTable.getSaveMethodValue}" + ) } case _ => throw new UnsupportedOperationException( "WriteOperation:SaveTypeCase not supported " - + s"${writeOperation.getSaveTypeCase.getNumber}") + + s"${writeOperation.getSaveTypeCase.getNumber}" + ) } executeHolder.eventsManager.postFinished() } @@ -3176,8 +3449,9 @@ class SparkConnectPlanner( } if (writeOperation.getTablePropertiesCount > 0) { - writeOperation.getTablePropertiesMap.asScala.foreach { case (key, value) => - w.tableProperty(key, value) + writeOperation.getTablePropertiesMap.asScala.foreach { + case (key, value) => + w.tableProperty(key, value) } } @@ -3221,7 +3495,8 @@ class SparkConnectPlanner( } case _ => throw new UnsupportedOperationException( - s"WriteOperationV2:ModeValue not supported ${writeOperation.getModeValue}") + s"WriteOperationV2:ModeValue not supported ${writeOperation.getModeValue}" + ) } executeHolder.eventsManager.postFinished() } @@ -3276,13 +3551,14 @@ class SparkConnectPlanner( val foreach = writeOp.getForeachWriter.getPythonFunction val pythonFcn = transformPythonFunction(foreach) writer.foreachImplementation( - new PythonForeachWriter(pythonFcn, dataset.schema).asInstanceOf[ForeachWriter[Any]]) + new PythonForeachWriter(pythonFcn, dataset.schema).asInstanceOf[ForeachWriter[Any]] + ) } else { val foreachWriterPkt = unpackForeachWriter(writeOp.getForeachWriter.getScalaFunction) val clientWriter = foreachWriterPkt.foreachWriter val encoder: Option[ExpressionEncoder[Any]] = Try( - ExpressionEncoder( - foreachWriterPkt.datasetEncoder.asInstanceOf[AgnosticEncoder[Any]])).toOption + ExpressionEncoder(foreachWriterPkt.datasetEncoder.asInstanceOf[AgnosticEncoder[Any]]) + ).toOption writer.foreachImplementation(clientWriter.asInstanceOf[ForeachWriter[Any]], encoder) } } @@ -3302,7 +3578,8 @@ class SparkConnectPlanner( case StreamingForeachFunction.FunctionCase.SCALA_FUNCTION => StreamingForeachBatchHelper.scalaForeachBatchWrapper( writeOp.getForeachBatch.getScalaFunction.getPayload.toByteArray, - sessionHolder) + sessionHolder + ) case StreamingForeachFunction.FunctionCase.FUNCTION_NOT_SET => throw InvalidPlanInput("Unexpected foreachBatch function") // Unreachable @@ -3322,7 +3599,8 @@ class SparkConnectPlanner( case NonFatal(ex) => // Failed to start the query, clean up foreach runner if any. logInfo( log"Removing foreachBatch worker, query failed to start " + - log"for session ${MDC(SESSION_ID, sessionId)}.") + log"for session ${MDC(SESSION_ID, sessionId)}." + ) foreachBatchRunnerCleaner.foreach(_.close()) throw ex } @@ -3332,12 +3610,11 @@ class SparkConnectPlanner( sessionHolder, query, executeHolder.sparkSessionTags, - executeHolder.operationId) + executeHolder.operationId + ) // Register the runner with the query if Python foreachBatch is enabled. foreachBatchRunnerCleaner.foreach { cleaner => - sessionHolder.streamingForeachBatchRunnerCleanerCache.registerCleanerForQuery( - query, - cleaner) + sessionHolder.streamingForeachBatchRunnerCleanerCache.registerCleanerForQuery(query, cleaner) } executeHolder.eventsManager.postFinished() @@ -3348,7 +3625,8 @@ class SparkConnectPlanner( .newBuilder() .setId(query.id.toString) .setRunId(query.runId.toString) - .build()) + .build() + ) .setName(Option(query.name).getOrElse("")) // The query started event for this query is sent to the client, and is handled by @@ -3365,16 +3643,18 @@ class SparkConnectPlanner( // before the client side DataStreamWriter.start(). // This way we ensure that the onQueryStarted API is called before the start() call in Connect. val queryStartedEvent = Option( - sessionHolder.streamingServersideListenerHolder.streamingQueryStartedEventCache.remove( - query.runId.toString)) + sessionHolder.streamingServersideListenerHolder.streamingQueryStartedEventCache + .remove(query.runId.toString) + ) queryStartedEvent.foreach { logInfo( log"[SessionId: ${MDC(LogKeys.SESSION_ID, sessionId)}]" + - log"[UserId: ${MDC(LogKeys.USER_ID, userId)}] " + - log"[operationId: ${MDC(LogKeys.OPERATION_ID, executeHolder.operationId)}] " + - log"[query id: ${MDC(LogKeys.QUERY_ID, query.id)}]" + - log"[query runId: ${MDC(LogKeys.QUERY_RUN_ID, query.runId)}] " + - log"Adding QueryStartedEvent to response") + log"[UserId: ${MDC(LogKeys.USER_ID, userId)}] " + + log"[operationId: ${MDC(LogKeys.OPERATION_ID, executeHolder.operationId)}] " + + log"[query id: ${MDC(LogKeys.QUERY_ID, query.id)}]" + + log"[query runId: ${MDC(LogKeys.QUERY_RUN_ID, query.runId)}] " + + log"Adding QueryStartedEvent to response" + ) e => resultBuilder.setQueryStartedEventJson(e.json) } @@ -3384,7 +3664,8 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setWriteStreamOperationStartResult(resultBuilder.build()) - .build()) + .build() + ) } private def handleStreamingQueryCommand( @@ -3400,7 +3681,7 @@ class SparkConnectPlanner( // Find the query in connect service level cache, otherwise check session's active streams. val query = SparkConnectService.streamingSessionManager - // Common case: query is cached in the cache. + // Common case: query is cached in the cache. .getCachedQuery(id, runId, executeHolder.sparkSessionTags, session) .map(_.query) .orElse { // Else try to find it in active streams. Mostly will not be found here either. @@ -3411,7 +3692,8 @@ class SparkConnectPlanner( case Some(query) => throw new IllegalArgumentException( s"Run id mismatch for query id $id. Run id in the request $runId " + - s"does not match one on the server ${query.runId}. The query might have restarted.") + s"does not match one on the server ${query.runId}. The query might have restarted." + ) case None => throw new IllegalArgumentException(s"Streaming query $id is not found") } @@ -3440,9 +3722,9 @@ class SparkConnectPlanner( respBuilder.setRecentProgress( StreamingQueryCommandResult.RecentProgressResult .newBuilder() - .addAllRecentProgressJson( - progressReports.map(StreamingQueryProgress.jsonString).asJava) - .build()) + .addAllRecentProgressJson(progressReports.map(StreamingQueryProgress.jsonString).asJava) + .build() + ) case StreamingQueryCommand.CommandCase.STOP => query.stop() @@ -3507,7 +3789,8 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setStreamingQueryCommandResult(respBuilder.build()) - .build()) + .build() + ) } /** @@ -3560,7 +3843,8 @@ class SparkConnectPlanner( .newBuilder() .setId(query.id.toString) .setRunId(query.runId.toString) - .build()) + .build() + ) if (query.name != null) { builder.setName(query.name) } @@ -3579,7 +3863,8 @@ class SparkConnectPlanner( active_queries .map(query => buildStreamingQueryInstance(query)) .toImmutableArraySeq - .asJava) + .asJava + ) case StreamingQueryManagerCommand.CommandCase.GET_QUERY => Option(session.streams.get(command.getGetQuery)).foreach { q => @@ -3604,12 +3889,14 @@ class SparkConnectPlanner( val listener = if (command.getAddListener.hasPythonListenerPayload) { new PythonStreamingQueryListener( transformPythonFunction(command.getAddListener.getPythonListenerPayload), - sessionHolder) + sessionHolder + ) } else { val listenerPacket = Utils .deserialize[StreamingListenerPacket]( command.getAddListener.getListenerPayload.toByteArray, - Utils.getContextOrSparkClassLoader) + Utils.getContextOrSparkClassLoader + ) listenerPacket.listener.asInstanceOf[StreamingQueryListener] } @@ -3645,7 +3932,8 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setStreamingQueryManagerCommandResult(respBuilder.build()) - .build()) + .build() + ) } private def handleGetResourcesCommand( @@ -3661,30 +3949,38 @@ class SparkConnectPlanner( .newBuilder() .putAllResources( session.sparkContext.resources.toMap - .transform((_, resource) => - proto.ResourceInformation - .newBuilder() - .setName(resource.name) - .addAllAddresses(resource.addresses.toImmutableArraySeq.asJava) - .build()) - .asJava) - .build()) - .build()) + .transform( + (_, resource) => + proto.ResourceInformation + .newBuilder() + .setName(resource.name) + .addAllAddresses(resource.addresses.toImmutableArraySeq.asJava) + .build() + ) + .asJava + ) + .build() + ) + .build() + ) } private def handleCreateResourceProfileCommand( createResourceProfileCommand: CreateResourceProfileCommand, responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { val rp = createResourceProfileCommand.getProfile - val ereqs = rp.getExecutorResourcesMap.asScala.map { case (name, res) => - name -> new ExecutorResourceRequest( - res.getResourceName, - res.getAmount, - res.getDiscoveryScript, - res.getVendor) + val ereqs = rp.getExecutorResourcesMap.asScala.map { + case (name, res) => + name -> new ExecutorResourceRequest( + res.getResourceName, + res.getAmount, + res.getDiscoveryScript, + res.getVendor + ) }.toMap - val treqs = rp.getTaskResourcesMap.asScala.map { case (name, res) => - name -> new TaskResourceRequest(res.getResourceName, res.getAmount) + val treqs = rp.getTaskResourcesMap.asScala.map { + case (name, res) => + name -> new TaskResourceRequest(res.getResourceName, res.getAmount) }.toMap // Create ResourceProfile add add it to ResourceProfileManager @@ -3705,8 +4001,10 @@ class SparkConnectPlanner( proto.CreateResourceProfileCommandResult .newBuilder() .setProfileId(profile.id) - .build()) - .build()) + .build() + ) + .build() + ) } private def handleCheckpointCommand( @@ -3719,7 +4017,8 @@ class SparkConnectPlanner( target.localCheckpoint( eager = checkpointCommand.getEager, storageLevel = - StorageLevelProtoConverter.toStorageLevel(checkpointCommand.getStorageLevel)) + StorageLevelProtoConverter.toStorageLevel(checkpointCommand.getStorageLevel) + ) } else { target.localCheckpoint(eager = checkpointCommand.getEager) } @@ -3741,8 +4040,10 @@ class SparkConnectPlanner( proto.CheckpointCommandResult .newBuilder() .setRelation(proto.CachedRemoteRelation.newBuilder().setRelationId(dfId).build()) - .build()) - .build()) + .build() + ) + .build() + ) } private def handleRemoveCachedRemoteRelationCommand( @@ -3777,7 +4078,8 @@ class SparkConnectPlanner( private val emptyLocalRelation = LocalRelation( output = AttributeReference("value", StringType, false)() :: Nil, - data = Seq.empty) + data = Seq.empty + ) private def transformCurrentDatabase(): LogicalPlan = { session.createDataset(session.catalog.currentDatabase :: Nil)(Encoders.STRING).logicalPlan @@ -3849,10 +4151,10 @@ class SparkConnectPlanner( if (getGetTable.hasDbName) { Catalog .makeDataset( - session.catalog.getTable( - dbName = getGetTable.getDbName, - tableName = getGetTable.getTableName) :: Nil, - session) + session.catalog + .getTable(dbName = getGetTable.getDbName, tableName = getGetTable.getTableName) :: Nil, + session + ) .logicalPlan } else { Catalog @@ -3867,8 +4169,10 @@ class SparkConnectPlanner( .makeDataset( session.catalog.getFunction( dbName = getGetFunction.getDbName, - functionName = getGetFunction.getFunctionName) :: Nil, - session) + functionName = getGetFunction.getFunctionName + ) :: Nil, + session + ) .logicalPlan } else { Catalog @@ -3880,7 +4184,8 @@ class SparkConnectPlanner( private def transformDatabaseExists(getDatabaseExists: proto.DatabaseExists): LogicalPlan = { session .createDataset(session.catalog.databaseExists(getDatabaseExists.getDbName) :: Nil)( - Encoders.scalaBoolean) + Encoders.scalaBoolean + ) .logicalPlan } @@ -3890,12 +4195,15 @@ class SparkConnectPlanner( .createDataset( session.catalog.tableExists( dbName = getTableExists.getDbName, - tableName = getTableExists.getTableName) :: Nil)(Encoders.scalaBoolean) + tableName = getTableExists.getTableName + ) :: Nil + )(Encoders.scalaBoolean) .logicalPlan } else { session .createDataset(session.catalog.tableExists(getTableExists.getTableName) :: Nil)( - Encoders.scalaBoolean) + Encoders.scalaBoolean + ) .logicalPlan } } @@ -3906,12 +4214,15 @@ class SparkConnectPlanner( .createDataset( session.catalog.functionExists( dbName = getFunctionExists.getDbName, - functionName = getFunctionExists.getFunctionName) :: Nil)(Encoders.scalaBoolean) + functionName = getFunctionExists.getFunctionName + ) :: Nil + )(Encoders.scalaBoolean) .logicalPlan } else { session .createDataset(session.catalog.functionExists(getFunctionExists.getFunctionName) :: Nil)( - Encoders.scalaBoolean) + Encoders.scalaBoolean + ) .logicalPlan } } @@ -3934,7 +4245,7 @@ class SparkConnectPlanner( val options = if (getCreateExternalTable.hasPath) { (getCreateExternalTable.getOptionsMap.asScala ++ - Map("path" -> getCreateExternalTable.getPath)).asJava + Map("path" -> getCreateExternalTable.getPath)).asJava } else { getCreateExternalTable.getOptionsMap } @@ -3943,7 +4254,8 @@ class SparkConnectPlanner( tableName = getCreateExternalTable.getTableName, source = source, schema = schema, - options = options) + options = options + ) .logicalPlan } @@ -3970,7 +4282,7 @@ class SparkConnectPlanner( val options = if (getCreateTable.hasPath) { (getCreateTable.getOptionsMap.asScala ++ - Map("path" -> getCreateTable.getPath)).asJava + Map("path" -> getCreateTable.getPath)).asJava } else { getCreateTable.getOptionsMap } @@ -3981,23 +4293,25 @@ class SparkConnectPlanner( source = source, schema = schema, description = description, - options = options) + options = options + ) .logicalPlan } private def transformDropTempView(getDropTempView: proto.DropTempView): LogicalPlan = { session .createDataset(session.catalog.dropTempView(getDropTempView.getViewName) :: Nil)( - Encoders.scalaBoolean) + Encoders.scalaBoolean + ) .logicalPlan } private def transformDropGlobalTempView( getDropGlobalTempView: proto.DropGlobalTempView): LogicalPlan = { session - .createDataset( - session.catalog.dropGlobalTempView(getDropGlobalTempView.getViewName) :: Nil)( - Encoders.scalaBoolean) + .createDataset(session.catalog.dropGlobalTempView(getDropGlobalTempView.getViewName) :: Nil)( + Encoders.scalaBoolean + ) .logicalPlan } @@ -4010,7 +4324,8 @@ class SparkConnectPlanner( private def transformIsCached(getIsCached: proto.IsCached): LogicalPlan = { session .createDataset(session.catalog.isCached(getIsCached.getTableName) :: Nil)( - Encoders.scalaBoolean) + Encoders.scalaBoolean + ) .logicalPlan } @@ -4018,7 +4333,8 @@ class SparkConnectPlanner( if (getCacheTable.hasStorageLevel) { session.catalog.cacheTable( getCacheTable.getTableName, - StorageLevelProtoConverter.toStorageLevel(getCacheTable.getStorageLevel)) + StorageLevelProtoConverter.toStorageLevel(getCacheTable.getStorageLevel) + ) } else { session.catalog.cacheTable(getCacheTable.getTableName) } @@ -4083,16 +4399,16 @@ class SparkConnectPlanner( .map(transformSortOrder) .toSeq, withSinglePartition = - options.hasWithSinglePartition && options.getWithSinglePartition) + options.hasWithSinglePartition && options.getWithSinglePartition + ) } else { UnresolvedTableArgPlanId(planId) } case proto.SubqueryExpression.SubqueryType.SUBQUERY_TYPE_IN => - UnresolvedInSubqueryPlanId( - getSubqueryExpression.getInSubqueryValuesList.asScala.map { value => + UnresolvedInSubqueryPlanId(getSubqueryExpression.getInSubqueryValuesList.asScala.map { + value => transformExpression(value) - }.toSeq, - planId) + }.toSeq, planId) case other => throw InvalidPlanInput(s"Unknown SubqueryType $other") } } @@ -4128,8 +4444,9 @@ class SparkConnectPlanner( assertPlan( missingPlanIds.isEmpty, "Missing relation in WithRelations: " + - s"${missingPlanIds.mkString("(", ", ", ")")} not in " + - s"${relations.keys.mkString("(", ", ", ")")}") + s"${missingPlanIds.mkString("(", ", ", ")")} not in " + + s"${relations.keys.mkString("(", ", ", ")")}" + ) withRelations } } From dcc3b68f5601c0224960260a6c7926ef95eadd2c Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Mon, 19 May 2025 11:44:54 -0700 Subject: [PATCH 2/9] 2 --- .../connect/planner/SparkConnectPlanner.scala | 891 ++++++------------ 1 file changed, 287 insertions(+), 604 deletions(-) diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 3e1b34224ace6..b5a24afe4ae35 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -19,166 +19,50 @@ package org.apache.spark.sql.connect.planner import java.util.Properties import java.util.UUID + import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.Try import scala.util.control.NonFatal + import com.google.common.base.Throwables import com.google.common.collect.{Lists, Maps} -import com.google.protobuf.{ByteString, Any => ProtoAny} +import com.google.protobuf.{Any => ProtoAny, ByteString} import io.grpc.{Context, Status, StatusRuntimeException} import io.grpc.stub.StreamObserver import org.apache.commons.lang3.exception.ExceptionUtils + import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction} import org.apache.spark.connect.proto -import org.apache.spark.connect.proto.{ - CheckpointCommand, - CreateResourceProfileCommand, - ExecutePlanResponse, - SqlCommand, - StreamingForeachFunction, - StreamingQueryCommand, - StreamingQueryCommandResult, - StreamingQueryInstanceId, - StreamingQueryManagerCommand, - StreamingQueryManagerCommandResult, - WriteStreamOperationStart, - WriteStreamOperationStartResult -} +import org.apache.spark.connect.proto.{CheckpointCommand, CreateResourceProfileCommand, ExecutePlanResponse, SqlCommand, StreamingForeachFunction, StreamingQueryCommand, StreamingQueryCommandResult, StreamingQueryInstanceId, StreamingQueryManagerCommand, StreamingQueryManagerCommandResult, WriteStreamOperationStart, WriteStreamOperationStartResult} import org.apache.spark.connect.proto.ExecutePlanResponse.SqlCommandResult import org.apache.spark.connect.proto.Parse.ParseFormat import org.apache.spark.connect.proto.StreamingQueryManagerCommandResult.StreamingQueryInstance import org.apache.spark.connect.proto.WriteStreamOperationStart.TriggerCase -import org.apache.spark.internal.{LogKeys, Logging, MDC} +import org.apache.spark.internal.{Logging, LogKeys, MDC} import org.apache.spark.internal.LogKeys.{DATAFRAME_ID, SESSION_ID} -import org.apache.spark.resource.{ - ExecutorResourceRequest, - ResourceProfile, - TaskResourceProfile, - TaskResourceRequest -} +import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile, TaskResourceProfile, TaskResourceRequest} import org.apache.spark.sql.{Column, Encoders, ForeachWriter, Observation, Row} -import org.apache.spark.sql.catalyst.{ - AliasIdentifier, - FunctionIdentifier, - QueryPlanningTracker, - expressions -} -import org.apache.spark.sql.catalyst.analysis.{ - FunctionRegistry, - GlobalTempView, - LocalTempView, - MultiAlias, - NameParameterizedQuery, - PosParameterizedQuery, - UnresolvedAlias, - UnresolvedAttribute, - UnresolvedDataFrameStar, - UnresolvedDeserializer, - UnresolvedExtractValue, - UnresolvedFunction, - UnresolvedOrdinal, - UnresolvedPlanId, - UnresolvedRegex, - UnresolvedRelation, - UnresolvedStar, - UnresolvedStarWithColumns, - UnresolvedStarWithColumnsRenames, - UnresolvedSubqueryColumnAliases, - UnresolvedTableValuedFunction, - UnresolvedTranspose -} -import org.apache.spark.sql.catalyst.encoders.{ - AgnosticEncoder, - ExpressionEncoder, - RowEncoder, - encoderFor -} -import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ - ProductEncoder, - StringEncoder, - UnboundRowEncoder, - RowEncoder => AgnosticRowEncoder -} +import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier, QueryPlanningTracker} +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedOrdinal, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedStarWithColumns, UnresolvedStarWithColumnsRenames, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunction, UnresolvedTranspose} +import org.apache.spark.sql.catalyst.encoders.{encoderFor, AgnosticEncoder, ExpressionEncoder, RowEncoder} +import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ProductEncoder, RowEncoder => AgnosticRowEncoder, StringEncoder, UnboundRowEncoder} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils} -import org.apache.spark.sql.catalyst.plans.{ - Cross, - FullOuter, - Inner, - JoinType, - LeftAnti, - LeftOuter, - LeftSemi, - RightOuter, - UsingJoin -} +import org.apache.spark.sql.catalyst.plans.{Cross, FullOuter, Inner, JoinType, LeftAnti, LeftOuter, LeftSemi, RightOuter, UsingJoin} import org.apache.spark.sql.catalyst.plans.logical -import org.apache.spark.sql.catalyst.plans.logical.{ - AppendColumns, - Assignment, - CoGroup, - CollectMetrics, - CommandResult, - Deduplicate, - DeduplicateWithinWatermark, - DeleteAction, - DeserializeToObject, - Except, - FlatMapGroupsWithState, - InsertAction, - InsertStarAction, - Intersect, - JoinWith, - LocalRelation, - LogicalGroupState, - LogicalPlan, - MapGroups, - MapPartitions, - MergeAction, - Project, - Sample, - SerializeFromObject, - Sort, - SubqueryAlias, - TimeModes, - TransformWithState, - TypedFilter, - Union, - Unpivot, - UnresolvedHint, - UpdateAction, - UpdateEventTimeWatermarkColumn, - UpdateStarAction -} +import org.apache.spark.sql.catalyst.plans.logical.{AppendColumns, Assignment, CoGroup, CollectMetrics, CommandResult, Deduplicate, DeduplicateWithinWatermark, DeleteAction, DeserializeToObject, Except, FlatMapGroupsWithState, InsertAction, InsertStarAction, Intersect, JoinWith, LocalRelation, LogicalGroupState, LogicalPlan, MapGroups, MapPartitions, MergeAction, Project, Sample, SerializeFromObject, Sort, SubqueryAlias, TimeModes, TransformWithState, TypedFilter, Union, Unpivot, UnresolvedHint, UpdateAction, UpdateEventTimeWatermarkColumn, UpdateStarAction} import org.apache.spark.sql.catalyst.streaming.InternalOutputModes import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin, TreePattern} import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, CharVarcharUtils} -import org.apache.spark.sql.classic.{ - Catalog, - Dataset, - MergeIntoWriter, - RelationalGroupedDataset, - SparkSession, - TypedAggUtils, - UserDefinedFunctionUtils -} +import org.apache.spark.sql.classic.{Catalog, Dataset, MergeIntoWriter, RelationalGroupedDataset, SparkSession, TypedAggUtils, UserDefinedFunctionUtils} import org.apache.spark.sql.classic.ClassicConversions._ import org.apache.spark.sql.connect.client.arrow.ArrowSerializer -import org.apache.spark.sql.connect.common.{ - DataTypeProtoConverter, - ForeachWriterPacket, - InvalidCommandInput, - InvalidPlanInput, - LiteralValueProtoConverter, - StorageLevelProtoConverter, - StreamingListenerPacket, - UdfPacket -} +import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, ForeachWriterPacket, InvalidCommandInput, InvalidPlanInput, LiteralValueProtoConverter, StorageLevelProtoConverter, StreamingListenerPacket, UdfPacket} import org.apache.spark.sql.connect.config.Connect.CONNECT_GRPC_ARROW_MAX_BATCH_SIZE import org.apache.spark.sql.connect.ml.MLHandler import org.apache.spark.sql.connect.plugin.SparkConnectPluginRegistry @@ -191,31 +75,13 @@ import org.apache.spark.sql.execution.arrow.ArrowConverters import org.apache.spark.sql.execution.command.{CreateViewCommand, ExternalCommandExecutor} import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.execution.datasources.v2.python.UserDefinedPythonDataSource -import org.apache.spark.sql.execution.python.{ - UserDefinedPythonFunction, - UserDefinedPythonTableFunction -} +import org.apache.spark.sql.execution.python.{UserDefinedPythonFunction, UserDefinedPythonTableFunction} import org.apache.spark.sql.execution.python.streaming.PythonForeachWriter import org.apache.spark.sql.execution.stat.StatFunctions import org.apache.spark.sql.execution.streaming.GroupStateImpl.groupStateTimeoutFromString import org.apache.spark.sql.execution.streaming.StreamingQueryWrapper -import org.apache.spark.sql.expressions.{ - Aggregator, - ReduceAggregator, - SparkUserDefinedFunction, - UserDefinedAggregator, - UserDefinedFunction -} -import org.apache.spark.sql.streaming.{ - GroupStateTimeout, - OutputMode, - StatefulProcessor, - StatefulProcessorWithInitialState, - StreamingQuery, - StreamingQueryListener, - StreamingQueryProgress, - Trigger -} +import org.apache.spark.sql.expressions.{Aggregator, ReduceAggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction} +import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode, StatefulProcessor, StatefulProcessorWithInitialState, StreamingQuery, StreamingQueryListener, StreamingQueryProgress, Trigger} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.{ArrowUtils, CaseInsensitiveStringMap} import org.apache.spark.storage.CacheId @@ -231,9 +97,7 @@ class SparkConnectPlanner( this(executeHolder.sessionHolder, Some(executeHolder)) } - if (!executeHolderOpt.forall { e => - e.sessionHolder == sessionHolder - }) { + if (!executeHolderOpt.forall { e => e.sessionHolder == sessionHolder }) { throw new IllegalArgumentException("executeHolder does not belong to sessionHolder") } @@ -349,7 +213,8 @@ class SparkConnectPlanner( case proto.Relation.RelTypeCase.APPLY_IN_PANDAS_WITH_STATE => transformApplyInPandasWithState(rel.getApplyInPandasWithState) case proto.Relation.RelTypeCase.COMMON_INLINE_USER_DEFINED_TABLE_FUNCTION => - transformCommonInlineUserDefinedTableFunction(rel.getCommonInlineUserDefinedTableFunction) + transformCommonInlineUserDefinedTableFunction( + rel.getCommonInlineUserDefinedTableFunction) case proto.Relation.RelTypeCase.CACHED_REMOTE_RELATION => transformCachedRemoteRelation(rel.getCachedRemoteRelation) case proto.Relation.RelTypeCase.COLLECT_METRICS => @@ -379,9 +244,9 @@ class SparkConnectPlanner( private def transformRelationPlugin(extension: ProtoAny): LogicalPlan = { SparkConnectPluginRegistry.relationRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.transform(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_.isPresent) @@ -442,8 +307,7 @@ class SparkConnectPlanner( .showString(rel.getNumRows, rel.getTruncate, rel.getVertical) LocalRelation.fromProduct( output = AttributeReference("show_string", StringType, false)() :: Nil, - data = Tuple1.apply(showString) :: Nil - ) + data = Tuple1.apply(showString) :: Nil) } private def transformHtmlString(rel: proto.HtmlString): LogicalPlan = { @@ -452,8 +316,7 @@ class SparkConnectPlanner( .htmlString(rel.getNumRows, rel.getTruncate) LocalRelation.fromProduct( output = AttributeReference("html_string", StringType, false)() :: Nil, - data = Tuple1.apply(htmlString) :: Nil - ) + data = Tuple1.apply(htmlString) :: Nil) } private def transformSql(sql: proto.SQL): LogicalPlan = { @@ -465,15 +328,13 @@ class SparkConnectPlanner( if (!namedArguments.isEmpty) { NameParameterizedQuery( parsedPlan, - namedArguments.asScala.toMap.transform((_, v) => transformExpression(v)) - ) + namedArguments.asScala.toMap.transform((_, v) => transformExpression(v))) } else if (!posArguments.isEmpty) { PosParameterizedQuery(parsedPlan, posArguments.asScala.map(transformExpression).toSeq) } else if (!args.isEmpty) { NameParameterizedQuery( parsedPlan, - args.asScala.toMap.transform((_, v) => transformLiteral(v)) - ) + args.asScala.toMap.transform((_, v) => transformLiteral(v))) } else if (!posArgs.isEmpty) { PosParameterizedQuery(parsedPlan, posArgs.asScala.map(transformLiteral).toSeq) } else { @@ -531,8 +392,7 @@ class SparkConnectPlanner( rel.getUpperBound, rel.getWithReplacement, if (rel.hasSeed) rel.getSeed else Utils.random.nextLong, - plan - ) + plan) } private def transformRepartition(rel: proto.Repartition): LogicalPlan = { @@ -558,8 +418,7 @@ class SparkConnectPlanner( if (rel.getValuesCount > 1 && rel.getValuesCount != rel.getColsCount) { throw InvalidPlanInput( s"When values contains more than 1 items, " + - s"values and cols should have the same length!" - ) + s"values and cols should have the same length!") } val dataset = Dataset.ofRows(session, transformRelation(rel.getInput)) @@ -572,9 +431,8 @@ class SparkConnectPlanner( dataset.na.fillValue(value, columns).logicalPlan } else { val valueMap = mutable.Map.empty[String, Any] - cols.zip(values).foreach { - case (col, value) => - valueMap.update(col, LiteralValueProtoConverter.toCatalystValue(value)) + cols.zip(values).foreach { case (col, value) => + valueMap.update(col, LiteralValueProtoConverter.toCatalystValue(value)) } dataset.na.fill(valueMap = valueMap.toMap).logicalPlan } @@ -602,8 +460,7 @@ class SparkConnectPlanner( rel.getReplacementsList.asScala.foreach { replace => replacement.update( LiteralValueProtoConverter.toCatalystValue(replace.getOldValue), - LiteralValueProtoConverter.toCatalystValue(replace.getNewValue) - ) + LiteralValueProtoConverter.toCatalystValue(replace.getNewValue)) } if (rel.getColsCount == 0) { @@ -665,8 +522,7 @@ class SparkConnectPlanner( LocalRelation.fromProduct( output = AttributeReference("approx_quantile", ArrayType(ArrayType(DoubleType)), false)() :: Nil, - data = Tuple1.apply(approxQuantile) :: Nil - ) + data = Tuple1.apply(approxQuantile) :: Nil) } private def transformStatCrosstab(rel: proto.StatCrosstab): LogicalPlan = { @@ -702,10 +558,7 @@ class SparkConnectPlanner( .sampleBy( col = Column(transformExpression(rel.getCol)), fractions = fractions.toMap, - seed = - if (rel.hasSeed) rel.getSeed - else Utils.random.nextLong - ) + seed = if (rel.hasSeed) rel.getSeed else Utils.random.nextLong) .logicalPlan } @@ -722,8 +575,7 @@ class SparkConnectPlanner( private def transformToDF(rel: proto.ToDF): LogicalPlan = { UnresolvedSubqueryColumnAliases( rel.getColumnNamesList.asScala.toSeq, - transformRelation(rel.getInput) - ) + transformRelation(rel.getInput)) } private def transformMapPartitions(rel: proto.MapPartitions): LogicalPlan = { @@ -749,25 +601,21 @@ class SparkConnectPlanner( DataTypeUtils.toAttributes(pythonUdf.dataType.asInstanceOf[StructType]), baseRel, isBarrier, - profile - ) + profile) case PythonEvalType.SQL_MAP_ARROW_ITER_UDF => logical.MapInArrow( pythonUdf, DataTypeUtils.toAttributes(pythonUdf.dataType.asInstanceOf[StructType]), baseRel, isBarrier, - profile - ) + profile) case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported" - ) + s"Function with EvalType: ${pythonUdf.evalType} is not supported") } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") } } @@ -785,8 +633,7 @@ class SparkConnectPlanner( val mapped = MapPartitions( udf.function.asInstanceOf[Iterator[Any] => Iterator[Any]], udf.outputObjAttr, - deserialized - ) + deserialized) SerializeFromObject(udf.outputNamedExpression, mapped) } @@ -799,8 +646,8 @@ class SparkConnectPlanner( case proto.CommonInlineUserDefinedFunction.FunctionCase.PYTHON_UDF => val pythonUdf = transformPythonUDF(commonUdf) val cols = - rel.getGroupingExpressionsList.asScala.toSeq - .map(expr => Column(transformExpression(expr))) + rel.getGroupingExpressionsList.asScala.toSeq.map(expr => + Column(transformExpression(expr))) val group = Dataset .ofRows(session, transformRelation(rel.getInput)) .groupBy(cols: _*) @@ -822,14 +669,12 @@ class SparkConnectPlanner( case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported" - ) + s"Function with EvalType: ${pythonUdf.evalType} is not supported") } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") } } @@ -841,8 +686,7 @@ class SparkConnectPlanner( val ds = UntypedKeyValueGroupedDataset( rel.getInput, rel.getGroupingExpressionsList, - rel.getSortingExpressionsList - ) + rel.getSortingExpressionsList) if (rel.hasTransformWithStateInfo) { val hasInitialState = !rel.getInitialGroupingExpressionsList.isEmpty && rel.hasInitialInput @@ -860,8 +704,7 @@ class SparkConnectPlanner( val initDs = UntypedKeyValueGroupedDataset( rel.getInitialInput, rel.getInitialGroupingExpressionsList, - rel.getSortingExpressionsList - ) + rel.getSortingExpressionsList) new TransformWithState( keyDeserializer, ds.valueDeserializer, @@ -877,8 +720,7 @@ class SparkConnectPlanner( initDs.groupingAttributes, initDs.dataAttributes, initDs.valueDeserializer, - initDs.analyzed - ) + initDs.analyzed) } else { val statefulProcessor = unpackedUdf.function.asInstanceOf[StatefulProcessor[Any, Any, Any]] @@ -897,8 +739,7 @@ class SparkConnectPlanner( ds.groupingAttributes, ds.dataAttributes, keyDeserializer, - LocalRelation(ds.vEncoder.schema) - ) + LocalRelation(ds.vEncoder.schema)) } val serializedPlan = SerializeFromObject(udf.outputNamedExpression, twsNode) @@ -906,8 +747,7 @@ class SparkConnectPlanner( val eventTimeWrappedPlan = UpdateEventTimeWatermarkColumn( UnresolvedAttribute(twsInfo.getEventTimeColumnName), None, - serializedPlan - ) + serializedPlan) eventTimeWrappedPlan } else { serializedPlan @@ -918,14 +758,12 @@ class SparkConnectPlanner( UntypedKeyValueGroupedDataset( rel.getInitialInput, rel.getInitialGroupingExpressionsList, - rel.getSortingExpressionsList - ) + rel.getSortingExpressionsList) } else { UntypedKeyValueGroupedDataset( rel.getInput, rel.getGroupingExpressionsList, - rel.getSortingExpressionsList - ) + rel.getSortingExpressionsList) } val timeoutConf = if (!rel.hasTimeoutConf) { GroupStateTimeout.NoTimeout @@ -941,14 +779,14 @@ class SparkConnectPlanner( val stateSchema = DataTypeProtoConverter.toCatalystType(rel.getStateSchema) match { case s: StructType => s case other => - throw InvalidPlanInput(s"Invalid state schema dataType $other for flatMapGroupsWithState") + throw InvalidPlanInput( + s"Invalid state schema dataType $other for flatMapGroupsWithState") } val stateEncoder = TypedScalaUdf.encoderFor( // the state agnostic encoder is the second element in the input encoders. unpackedUdf.inputEncoders.tail.head, "state", - Some(DataTypeUtils.toAttributes(stateSchema)) - ) + Some(DataTypeUtils.toAttributes(stateSchema))) val flatMapGroupsWithState = if (hasInitialState) { new FlatMapGroupsWithState( @@ -968,8 +806,7 @@ class SparkConnectPlanner( initialDs.dataAttributes, initialDs.valueDeserializer, initialDs.analyzed, - ds.analyzed - ) + ds.analyzed) } else { new FlatMapGroupsWithState( udf.function @@ -988,8 +825,7 @@ class SparkConnectPlanner( ds.dataAttributes, udf.inputDeserializer(ds.groupingAttributes), LocalRelation(initialDs.vEncoder.schema), // empty data set - ds.analyzed - ) + ds.analyzed) } SerializeFromObject(udf.outputNamedExpression, flatMapGroupsWithState) } else { @@ -1001,8 +837,7 @@ class SparkConnectPlanner( ds.dataAttributes, ds.sortOrder, udf.outputObjAttr, - ds.analyzed - ) + ds.analyzed) SerializeFromObject(udf.outputNamedExpression, mapped) } } @@ -1015,11 +850,11 @@ class SparkConnectPlanner( case proto.CommonInlineUserDefinedFunction.FunctionCase.PYTHON_UDF => val inputCols = - rel.getInputGroupingExpressionsList.asScala.toSeq - .map(expr => Column(transformExpression(expr))) + rel.getInputGroupingExpressionsList.asScala.toSeq.map(expr => + Column(transformExpression(expr))) val otherCols = - rel.getOtherGroupingExpressionsList.asScala.toSeq - .map(expr => Column(transformExpression(expr))) + rel.getOtherGroupingExpressionsList.asScala.toSeq.map(expr => + Column(transformExpression(expr))) val input = Dataset .ofRows(session, transformRelation(rel.getInput)) @@ -1041,14 +876,12 @@ class SparkConnectPlanner( case _ => throw InvalidPlanInput( - s"Function with EvalType: ${pythonUdf.evalType} is not supported" - ) + s"Function with EvalType: ${pythonUdf.evalType} is not supported") } case _ => throw InvalidPlanInput( - s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${commonUdf.getFunctionCase.getNumber} is not supported") } } @@ -1059,13 +892,11 @@ class SparkConnectPlanner( val left = UntypedKeyValueGroupedDataset( rel.getInput, rel.getInputGroupingExpressionsList, - rel.getInputSortingExpressionsList - ) + rel.getInputSortingExpressionsList) val right = UntypedKeyValueGroupedDataset( rel.getOther, rel.getOtherGroupingExpressionsList, - rel.getOtherSortingExpressionsList - ) + rel.getOtherSortingExpressionsList) val mapped = CoGroup( udf.function.asInstanceOf[(Any, Iterator[Any], Iterator[Any]) => IterableOnce[Any]], @@ -1082,8 +913,7 @@ class SparkConnectPlanner( right.sortOrder, udf.outputObjAttr, left.analyzed, - right.analyzed - ) + right.analyzed) SerializeFromObject(udf.outputNamedExpression, mapped) } @@ -1129,8 +959,7 @@ class SparkConnectPlanner( createFromRelationalDataset(logicalPlan, groupingExprs, sortOrder) } else { throw InvalidPlanInput( - "The grouping expression cannot be absent for KeyValueGroupedDataset" - ) + "The grouping expression cannot be absent for KeyValueGroupedDataset") } } @@ -1153,8 +982,7 @@ class SparkConnectPlanner( qe.analyzed, analyzed, aliasedGroupings, - sortOrder - ) + sortOrder) } private def createFromGroupByKeyFunc( @@ -1178,8 +1006,7 @@ class SparkConnectPlanner( withGroupingKeyAnalyzed, analyzed, withGroupingKey.newColumns, - sortOrder - ) + sortOrder) } } @@ -1235,17 +1062,11 @@ class SparkConnectPlanner( inputAttrs: Option[Seq[Attribute]] = None): ExpressionEncoder[_] = { if (encoder == UnboundRowEncoder) { inputAttrs - .map( - attrs => - ExpressionEncoder( - RowEncoder.encoderFor( - StructType(attrs.map(a => StructField(a.name, a.dataType, a.nullable))) - ) - ) - ) + .map(attrs => + ExpressionEncoder(RowEncoder.encoderFor(StructType(attrs.map(a => + StructField(a.name, a.dataType, a.nullable)))))) .getOrElse( - throw InvalidPlanInput(s"Row is not a supported $errorType type for this UDF.") - ) + throw InvalidPlanInput(s"Row is not a supported $errorType type for this UDF.")) } else { // Nested unbound row encoders val unboundTransformed = encoder match { @@ -1281,8 +1102,7 @@ class SparkConnectPlanner( outputSchema, stateSchema, rel.getOutputMode, - rel.getTimeoutConf - ) + rel.getTimeoutConf) .logicalPlan } @@ -1298,14 +1118,13 @@ class SparkConnectPlanner( case dt => throw InvalidPlanInput( "Invalid user-defined output schema type for TransformWithStateInPandas. " + - s"Expect a struct type, but got ${dt.typeName}." - ) + s"Expect a struct type, but got ${dt.typeName}.") } } if (rel.hasInitialInput) { - val initialGroupingCols = rel.getInitialGroupingExpressionsList.asScala.toSeq - .map(expr => Column(transformExpression(expr))) + val initialGroupingCols = rel.getInitialGroupingExpressionsList.asScala.toSeq.map(expr => + Column(transformExpression(expr))) val initialStateDs = Dataset .ofRows(session, transformRelation(rel.getInitialInput)) @@ -1325,8 +1144,7 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, initialStateDs, - twsInfo.getEventTimeColumnName - ) + twsInfo.getEventTimeColumnName) .logicalPlan } else { // use Row @@ -1337,8 +1155,7 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, initialStateDs, - twsInfo.getEventTimeColumnName - ) + twsInfo.getEventTimeColumnName) .logicalPlan } @@ -1351,8 +1168,7 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, null, - twsInfo.getEventTimeColumnName - ) + twsInfo.getEventTimeColumnName) .logicalPlan } else { // use Row @@ -1363,8 +1179,7 @@ class SparkConnectPlanner( rel.getOutputMode, twsInfo.getTimeMode, null, - twsInfo.getEventTimeColumnName - ) + twsInfo.getEventTimeColumnName) .logicalPlan } } @@ -1377,12 +1192,10 @@ class SparkConnectPlanner( val function = createPythonUserDefinedTableFunction(fun) function.builder( fun.getArgumentsList.asScala.map(transformExpression).toSeq, - session.sessionState.sqlParser - ) + session.sessionState.sqlParser) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") } } @@ -1397,8 +1210,7 @@ class SparkConnectPlanner( // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull - ) + accumulator = sessionHolder.pythonAccumulator.orNull) } private def transformPythonDataSource(ds: proto.PythonDataSource): SimplePythonFunction = { @@ -1412,8 +1224,7 @@ class SparkConnectPlanner( // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull - ) + accumulator = sessionHolder.pythonAccumulator.orNull) } private def transformCachedRemoteRelation(rel: proto.CachedRemoteRelation): LogicalPlan = { @@ -1433,8 +1244,7 @@ class SparkConnectPlanner( } Project( Seq(UnresolvedStarWithColumnsRenames(existingNames = colNames, newNames = newColNames)), - transformRelation(rel.getInput) - ) + transformRelation(rel.getInput)) } private def transformWithColumns(rel: proto.WithColumns): LogicalPlan = { @@ -1459,11 +1269,8 @@ class SparkConnectPlanner( UnresolvedStarWithColumns( colNames = colNames, exprs = exprs, - explicitMetadata = Some(metadata) - ) - ), - transformRelation(rel.getInput) - ) + explicitMetadata = Some(metadata))), + transformRelation(rel.getInput)) } private def transformWithWatermark(rel: proto.WithWatermark): LogicalPlan = { @@ -1493,8 +1300,7 @@ class SparkConnectPlanner( .getOrElse { throw InvalidPlanInput( s"Not found any cached local relation with the hash: ${blockId.hash} in " + - s"the session with sessionUUID ${blockId.sessionUUID}." - ) + s"the session with sessionUUID ${blockId.sessionUUID}.") } } @@ -1515,8 +1321,7 @@ class SparkConnectPlanner( rel: proto.UnresolvedTableValuedFunction): LogicalPlan = { UnresolvedTableValuedFunction( rel.getFunctionName, - rel.getArgumentsList.asScala.map(transformExpression).toSeq - ) + rel.getArgumentsList.asScala.map(transformExpression).toSeq) } private def transformUnpivot(rel: proto.Unpivot): LogicalPlan = { @@ -1531,8 +1336,7 @@ class SparkConnectPlanner( None, rel.getVariableColumnName, Seq(rel.getValueColumnName), - transformRelation(rel.getInput) - ) + transformRelation(rel.getInput)) } else { val values = rel.getValues.getValuesList.asScala.toArray.map { expr => Column(transformExpression(expr)) @@ -1544,12 +1348,12 @@ class SparkConnectPlanner( None, rel.getVariableColumnName, Seq(rel.getValueColumnName), - transformRelation(rel.getInput) - ) + transformRelation(rel.getInput)) } } - private def transformRepartitionByExpression(rel: proto.RepartitionByExpression): LogicalPlan = { + private def transformRepartitionByExpression( + rel: proto.RepartitionByExpression): LogicalPlan = { val numPartitionsOpt = if (rel.hasNumPartitions) { Some(rel.getNumPartitions) } else { @@ -1559,8 +1363,7 @@ class SparkConnectPlanner( logical.RepartitionByExpression( partitionExpressions, transformRelation(rel.getInput), - numPartitionsOpt - ) + numPartitionsOpt) } private def transformCollectMetrics(rel: proto.CollectMetrics, planId: Long): LogicalPlan = { @@ -1591,8 +1394,7 @@ class SparkConnectPlanner( } if (!rel.getAllColumnsAsKeys && rel.getColumnNamesCount == 0) { throw InvalidPlanInput( - "Deduplicate requires to either deduplicate on all columns or a subset of columns" - ) + "Deduplicate requires to either deduplicate on all columns or a subset of columns") } val queryExecution = new QueryExecution(session, transformRelation(rel.getInput)) val resolver = session.sessionState.analyzer.resolver @@ -1649,8 +1451,7 @@ class SparkConnectPlanner( val schemaType = DataType.parseTypeWithFallback( rel.getSchema, parseDatatypeString, - fallbackParser = DataType.fromJson - ) + fallbackParser = DataType.fromJson) schema = schemaType match { case s: StructType => s case d => StructType(Seq(StructField("value", d))) @@ -1660,8 +1461,7 @@ class SparkConnectPlanner( if (rel.hasData) { val (rows, structType) = ArrowConverters.fromBatchWithSchemaIterator( Iterator(rel.getData.toByteArray), - TaskContext.get() - ) + TaskContext.get()) if (structType == null) { throw InvalidPlanInput(s"Input data for LocalRelation does not produce a schema.") } @@ -1693,8 +1493,7 @@ class SparkConnectPlanner( val project = Dataset .ofRows( session, - logicalPlan = logical.LocalRelation(normalize(structType).asInstanceOf[StructType]) - ) + logicalPlan = logical.LocalRelation(normalize(structType).asInstanceOf[StructType])) .toDF(normalized.names.toImmutableArraySeq: _*) .to(normalized) .logicalPlan @@ -1703,14 +1502,12 @@ class SparkConnectPlanner( val proj = UnsafeProjection.create(project.projectList, project.child.output) logical.LocalRelation( DataTypeUtils.toAttributes(schema), - data.map(proj).map(_.copy()).toSeq - ) + data.map(proj).map(_.copy()).toSeq) } } else { if (schema == null) { throw InvalidPlanInput( - s"Schema for LocalRelation is required when the input data is not provided." - ) + s"Schema for LocalRelation is required when the input data is not provided.") } LocalRelation(schema) } @@ -1718,7 +1515,10 @@ class SparkConnectPlanner( /** Parse as DDL, with a fallback to JSON. Throws an exception if if fails to parse. */ private def parseSchema(schema: String): StructType = { - DataType.parseTypeWithFallback(schema, StructType.fromDDL, fallbackParser = DataType.fromJson) match { + DataType.parseTypeWithFallback( + schema, + StructType.fromDDL, + fallbackParser = DataType.fromJson) match { case s: StructType => s case other => throw InvalidPlanInput(s"Invalid schema $other") } @@ -1731,8 +1531,7 @@ class SparkConnectPlanner( UnresolvedRelation( parser.parseMultipartIdentifier(rel.getNamedTable.getUnparsedIdentifier), new CaseInsensitiveStringMap(rel.getNamedTable.getOptionsMap), - isStreaming = rel.getIsStreaming - ) + isStreaming = rel.getIsStreaming) case proto.Read.ReadTypeCase.DATA_SOURCE if !rel.getIsStreaming => val reader = session.read @@ -1766,8 +1565,7 @@ class SparkConnectPlanner( } } else { throw InvalidPlanInput( - s"Predicates are not supported for ${rel.getDataSource.getFormat} data sources." - ) + s"Predicates are not supported for ${rel.getDataSource.getFormat} data sources.") } case proto.Read.ReadTypeCase.DATA_SOURCE if rel.getIsStreaming => @@ -1834,9 +1632,9 @@ class SparkConnectPlanner( val udf = expr.getCommonInlineUserDefinedFunction // A typed scala udf is a scala udf && the udf argument is an unresolved start. udf.getFunctionCase == - proto.CommonInlineUserDefinedFunction.FunctionCase.SCALAR_SCALA_UDF && - udf.getArgumentsCount == 1 && - udf.getArguments(0).getExprTypeCase == proto.Expression.ExprTypeCase.UNRESOLVED_STAR + proto.CommonInlineUserDefinedFunction.FunctionCase.SCALAR_SCALA_UDF && + udf.getArgumentsCount == 1 && + udf.getArguments(0).getExprTypeCase == proto.Expression.ExprTypeCase.UNRESOLVED_STAR case _ => false } @@ -1858,8 +1656,8 @@ class SparkConnectPlanner( } val logicalPlan = - if (rel.getExpressionsList.asScala.toSeq - .exists(_.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { + if (rel.getExpressionsList.asScala.toSeq.exists( + _.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { session.sessionState.executePlan(baseRel).analyzed } else { baseRel @@ -1938,8 +1736,7 @@ class SparkConnectPlanner( Some(origin.getStackTraceList.asScala.map(transformStackTraceElement).toArray) } else { None - } - ) + }) } private def transformStackTraceElement(element: proto.StackTraceElement): StackTraceElement = { @@ -1950,8 +1747,7 @@ class SparkConnectPlanner( /* @NotNull declaringClass */ element.getDeclaringClass, /* @NotNull methodName */ element.getMethodName, /* fileName */ if (element.hasFileName) element.getFileName else null, - /* lineNumber */ element.getLineNumber - ) + /* lineNumber */ element.getLineNumber) } private def doTransformExpression( @@ -1998,8 +1794,7 @@ class SparkConnectPlanner( transformSubqueryExpression(exp.getSubqueryExpression) case _ => throw InvalidPlanInput( - s"Expression with ID: ${exp.getExprTypeCase.getNumber} is not supported" - ) + s"Expression with ID: ${exp.getExprTypeCase.getNumber} is not supported") } } @@ -2022,9 +1817,9 @@ class SparkConnectPlanner( private def transformExpressionPlugin(extension: ProtoAny): Expression = { SparkConnectPluginRegistry.expressionRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.transform(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_.isPresent) @@ -2044,34 +1839,31 @@ class SparkConnectPlanner( private def transformLimit(limit: proto.Limit): LogicalPlan = { logical.Limit( limitExpr = expressions.Literal(limit.getLimit, IntegerType), - transformRelation(limit.getInput) - ) + transformRelation(limit.getInput)) } private def transformTail(tail: proto.Tail): LogicalPlan = { logical.Tail( limitExpr = expressions.Literal(tail.getLimit, IntegerType), - transformRelation(tail.getInput) - ) + transformRelation(tail.getInput)) } private def transformOffset(offset: proto.Offset): LogicalPlan = { logical.Offset( offsetExpr = expressions.Literal(offset.getOffset, IntegerType), - transformRelation(offset.getInput) - ) + transformRelation(offset.getInput)) } /** * Translates a scalar function from proto to the Catalyst expression. */ - private def transformUnresolvedFunction(fun: proto.Expression.UnresolvedFunction): Expression = { + private def transformUnresolvedFunction( + fun: proto.Expression.UnresolvedFunction): Expression = { if (fun.getIsUserDefinedFunction) { UnresolvedFunction( parser.parseMultipartIdentifier(fun.getFunctionName), fun.getArgumentsList.asScala.map(transformExpression).toSeq, - isDistinct = fun.getIsDistinct - ) + isDistinct = fun.getIsDistinct) } else { val name = fun.getFunctionName val internal = if (fun.hasIsInternal) { @@ -2089,8 +1881,7 @@ class SparkConnectPlanner( name :: Nil, fun.getArgumentsList.asScala.map(transformExpression).toSeq, isDistinct = fun.getIsDistinct, - isInternal = internal - ) + isInternal = internal) } } @@ -2111,8 +1902,7 @@ class SparkConnectPlanner( transformScalaUDF(fun) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") } } @@ -2129,8 +1919,7 @@ class SparkConnectPlanner( UnresolvedFunction( parser.parseMultipartIdentifier(funcName), fun.getArgumentsList.asScala.map(transformExpression).toSeq, - false - ) + false) } private def transformNamedArgumentExpression( @@ -2156,15 +1945,13 @@ class SparkConnectPlanner( case nsm: NoSuchMethodException => throw new ClassNotFoundException( s"Failed to load class correctly due to $nsm. " + - "Make sure the artifact where the class is defined is installed by calling" + - " session.addArtifact." - ) + "Make sure the artifact where the class is defined is installed by calling" + + " session.addArtifact.") case cnf: ClassNotFoundException => throw new ClassNotFoundException( s"Failed to load class: ${cnf.getMessage}. " + - "Make sure the artifact where the class is defined is installed by calling" + - " session.addArtifact." - ) + "Make sure the artifact where the class is defined is installed by calling" + + " session.addArtifact.") case _ => throw t } } @@ -2187,7 +1974,8 @@ class SparkConnectPlanner( case uda: UserDefinedAggregator[_, _, _] => ScalaAggregator(uda, children).toAggregateExpression(fun.getIsDistinct) case other => - throw InvalidPlanInput(s"Unsupported UserDefinedFunction implementation: ${other.getClass}") + throw InvalidPlanInput( + s"Unsupported UserDefinedFunction implementation: ${other.getClass}") } } @@ -2211,8 +1999,7 @@ class SparkConnectPlanner( inputEncoder = ExpressionEncoder(udfPacket.inputEncoders.head), givenName = Option(fun.getFunctionName), nullable = udf.getNullable, - deterministic = fun.getDeterministic - ) + deterministic = fun.getDeterministic) } else { SparkUserDefinedFunction( f = udfPacket.function, @@ -2221,8 +2008,7 @@ class SparkConnectPlanner( outputEncoder = Option(ExpressionEncoder(udfPacket.outputEncoder)), givenName = Option(fun.getFunctionName), nullable = udf.getNullable, - deterministic = fun.getDeterministic - ) + deterministic = fun.getDeterministic) } } @@ -2256,8 +2042,7 @@ class SparkConnectPlanner( func = function, dataType = transformDataType(udf.getOutputType), pythonEvalType = udf.getEvalType, - udfDeterministic = fun.getDeterministic - ) + udfDeterministic = fun.getDeterministic) } private def transformPythonFunction(fun: proto.PythonUDF): SimplePythonFunction = { @@ -2268,13 +2053,12 @@ class SparkConnectPlanner( pythonExec = pythonExec, // Merge the user specified includes with the includes managed by the artifact manager. pythonIncludes = (fun.getAdditionalIncludesList.asScala.toSeq ++ - sessionHolder.artifactManager.getPythonIncludes).asJava, + sessionHolder.artifactManager.getPythonIncludes).asJava, pythonVer = fun.getPythonVer, // Empty broadcast variables broadcastVars = Lists.newArrayList(), // Accumulator if available - accumulator = sessionHolder.pythonAccumulator.orNull - ) + accumulator = sessionHolder.pythonAccumulator.orNull) } /** @@ -2284,15 +2068,13 @@ class SparkConnectPlanner( if (lambda.getArgumentsCount == 0 || lambda.getArgumentsCount > 3) { throw InvalidPlanInput( "LambdaFunction requires 1 ~ 3 arguments, " + - s"but got ${lambda.getArgumentsCount} ones!" - ) + s"but got ${lambda.getArgumentsCount} ones!") } LambdaFunction( function = transformExpression(lambda.getFunction), arguments = lambda.getArgumentsList.asScala.toSeq - .map(transformUnresolvedNamedLambdaVariable) - ) + .map(transformUnresolvedNamedLambdaVariable)) } private def transformUnresolvedNamedLambdaVariable( @@ -2315,8 +2097,7 @@ class SparkConnectPlanner( } else { if (alias.hasMetadata) { throw InvalidPlanInput( - "Alias expressions with more than 1 identifier must not use optional metadata." - ) + "Alias expressions with more than 1 identifier must not use optional metadata.") } MultiAlias(transformExpression(alias.getExpr), alias.getNameList.asScala.toSeq) } @@ -2337,8 +2118,7 @@ class SparkConnectPlanner( val target = star.getUnparsedTarget if (!target.endsWith(".*")) { throw InvalidPlanInput( - s"UnresolvedStar requires a unparsed target ending with '.*', but got $target." - ) + s"UnresolvedStar requires a unparsed target ending with '.*', but got $target.") } val parts = UnresolvedAttribute.parseAttributeName(target.dropRight(2)) UnresolvedStar(Some(parts)) @@ -2392,8 +2172,7 @@ class SparkConnectPlanner( extract: proto.Expression.UnresolvedExtractValue): UnresolvedExtractValue = { UnresolvedExtractValue( transformExpression(extract.getChild), - transformExpression(extract.getExtraction) - ) + transformExpression(extract.getExtraction)) } private def transformUpdateFields(update: proto.Expression.UpdateFields): UpdateFields = { @@ -2402,14 +2181,12 @@ class SparkConnectPlanner( UpdateFields.apply( col = transformExpression(update.getStructExpression), fieldName = update.getFieldName, - expr = transformExpression(update.getValueExpression) - ) + expr = transformExpression(update.getValueExpression)) } else { // drop a field UpdateFields.apply( col = transformExpression(update.getStructExpression), - fieldName = update.getFieldName - ) + fieldName = update.getFieldName) } } @@ -2470,13 +2247,11 @@ class SparkConnectPlanner( val windowSpec = WindowSpecDefinition( partitionSpec = window.getPartitionSpecList.asScala.toSeq.map(transformExpression), orderSpec = window.getOrderSpecList.asScala.toSeq.map(transformSortOrder), - frameSpecification = frameSpec - ) + frameSpecification = frameSpec) WindowExpression( windowFunction = transformExpression(window.getWindowFunction), - windowSpec = windowSpec - ) + windowSpec = windowSpec) } private def transformSetOperation(u: proto.SetOperation): LogicalPlan = { @@ -2501,8 +2276,7 @@ class SparkConnectPlanner( case proto.SetOperation.SetOpType.SET_OP_TYPE_UNION => if (!u.getByName && u.getAllowMissingColumns) { throw InvalidPlanInput( - "UnionByName `allowMissingCol` can be true only if `byName` is true." - ) + "UnionByName `allowMissingCol` can be true only if `byName` is true.") } val union = Union(Seq(leftPlan, rightPlan), u.getByName, u.getAllowMissingColumns) if (isAll) { @@ -2525,8 +2299,7 @@ class SparkConnectPlanner( session.sessionState.conf.dataFrameSelfJoinAutoResolveAmbiguity, session.sessionState.analyzer.resolver, rel.getJoinDataType.getIsLeftStruct, - rel.getJoinDataType.getIsRightStruct - ) + rel.getJoinDataType.getIsRightStruct) } private def transformJoinOrJoinWith(rel: proto.Join): LogicalPlan = { @@ -2541,14 +2314,12 @@ class SparkConnectPlanner( assertPlan(rel.hasLeft && rel.hasRight, "Both join sides must be present") if (rel.hasJoinCondition && rel.getUsingColumnsCount > 0) { throw InvalidPlanInput( - s"Using columns or join conditions cannot be set at the same time in Join" - ) + s"Using columns or join conditions cannot be set at the same time in Join") } val joinCondition = if (rel.hasJoinCondition) Some(transformExpression(rel.getJoinCondition)) else None val catalystJointype = transformJoinType( - if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER - ) + if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER) val joinType = if (rel.getUsingColumnsCount > 0) { UsingJoin(catalystJointype, rel.getUsingColumnsList.asScala.toSeq) } else { @@ -2559,8 +2330,7 @@ class SparkConnectPlanner( right = transformRelation(rel.getRight), joinType = joinType, condition = joinCondition, - hint = logical.JoinHint.NONE - ) + hint = logical.JoinHint.NONE) } private def transformJoinType(t: proto.Join.JoinType): JoinType = { @@ -2596,8 +2366,7 @@ class SparkConnectPlanner( joinType = joinType, tolerance = tolerance, allowExactMatches = allowExactMatches, - direction = direction - ) + direction = direction) } else { val joinExprs = if (rel.hasJoinExpr) Column(transformExpression(rel.getJoinExpr)) else null left.joinAsOf( @@ -2608,8 +2377,7 @@ class SparkConnectPlanner( joinType = joinType, tolerance = tolerance, allowExactMatches = allowExactMatches, - direction = direction - ) + direction = direction) } joined.logicalPlan } @@ -2619,14 +2387,12 @@ class SparkConnectPlanner( val joinCondition = if (rel.hasJoinCondition) Some(transformExpression(rel.getJoinCondition)) else None val joinType = transformJoinType( - if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER - ) + if (rel.getJoinType != null) rel.getJoinType else proto.Join.JoinType.JOIN_TYPE_INNER) logical.LateralJoin( left = transformRelation(rel.getLeft), right = LateralSubquery(transformRelation(rel.getRight)), joinType = joinType, - condition = joinCondition - ) + condition = joinCondition) } private def transformSort(sort: proto.Sort): LogicalPlan = { @@ -2634,8 +2400,7 @@ class SparkConnectPlanner( logical.Sort( child = transformRelation(sort.getInput), global = sort.getIsGlobal, - order = sort.getOrderList.asScala.toSeq.map(transformSortOrder) - ) + order = sort.getOrderList.asScala.toSeq.map(transformSortOrder)) } private def transformSortOrder(order: proto.Expression.SortOrder) = { @@ -2651,8 +2416,7 @@ class SparkConnectPlanner( expressions.NullsFirst case _ => expressions.NullsLast }, - sameOrderExpressions = Seq.empty - ) + sameOrderExpressions = Seq.empty) } /** @@ -2688,7 +2452,7 @@ class SparkConnectPlanner( // This is the case for datasets created via groupByKey, // and also via RelationalGroupedDS#as, as the first is a dummy UDF currently. if rel.getGroupingExpressionsList.size() >= 1 && - isTypedScalaUdfExpr(rel.getGroupingExpressionsList.get(0)) => + isTypedScalaUdfExpr(rel.getGroupingExpressionsList.get(0)) => transformKeyValueGroupedAggregate(rel) case _ => transformRelationalGroupedAggregate(rel) @@ -2713,8 +2477,8 @@ class SparkConnectPlanner( val input = transformRelation(rel.getInput) val logicalPlan = - if (rel.getAggregateExpressionsList.asScala.toSeq - .exists(_.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { + if (rel.getAggregateExpressionsList.asScala.toSeq.exists( + _.getExprTypeCase == proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION)) { session.sessionState.executePlan(input).analyzed } else { input @@ -2731,22 +2495,19 @@ class SparkConnectPlanner( logical.Aggregate( groupingExpressions = groupingExpressionsWithOrdinals, aggregateExpressions = aliasedAgg, - child = logicalPlan - ) + child = logicalPlan) case proto.Aggregate.GroupType.GROUP_TYPE_ROLLUP => logical.Aggregate( groupingExpressions = Seq(Rollup(groupingExpressionsWithOrdinals.map(Seq(_)))), aggregateExpressions = aliasedAgg, - child = logicalPlan - ) + child = logicalPlan) case proto.Aggregate.GroupType.GROUP_TYPE_CUBE => logical.Aggregate( groupingExpressions = Seq(Cube(groupingExpressionsWithOrdinals.map(Seq(_)))), aggregateExpressions = aliasedAgg, - child = logicalPlan - ) + child = logicalPlan) case proto.Aggregate.GroupType.GROUP_TYPE_PIVOT => if (!rel.hasPivot) { @@ -2766,8 +2527,7 @@ class SparkConnectPlanner( pivotColumn = pivotExpr, pivotValues = valueExprs, aggregates = aggExprs, - child = logicalPlan - ) + child = logicalPlan) case proto.Aggregate.GroupType.GROUP_TYPE_GROUPING_SETS => val groupingSetsExpressionsWithOrdinals = @@ -2779,12 +2539,9 @@ class SparkConnectPlanner( groupingExpressions = Seq( GroupingSets( groupingSets = groupingSetsExpressionsWithOrdinals, - userGivenGroupByExprs = groupingExpressionsWithOrdinals - ) - ), + userGivenGroupByExprs = groupingExpressionsWithOrdinals)), aggregateExpressions = aliasedAgg, - child = logicalPlan - ) + child = logicalPlan) case other => throw InvalidPlanInput(s"Unknown Group Type $other") } @@ -2813,7 +2570,7 @@ class SparkConnectPlanner( val udf = fun.getArgumentsList.asScala match { case collection.Seq(e) if e.hasCommonInlineUserDefinedFunction && - e.getCommonInlineUserDefinedFunction.hasScalarScalaUdf => + e.getCommonInlineUserDefinedFunction.hasScalarScalaUdf => unpackUdf(e.getCommonInlineUserDefinedFunction) case other => throw InvalidPlanInput(s"reduce should carry a scalar scala udf, but got $other") @@ -2850,8 +2607,10 @@ class SparkConnectPlanner( TypedAggregateExpression(aggregator)(aggregator.bufferEncoder, aggregator.outputEncoder) val taeWithInput = baseRelationOpt match { case Some(baseRelation) => - val inputEncoder = - TypedScalaUdf.encoderFor(udfPacket.inputEncoders.head, "input", Some(baseRelation.output)) + val inputEncoder = TypedScalaUdf.encoderFor( + udfPacket.inputEncoders.head, + "input", + Some(baseRelation.output)) TypedAggUtils .withInputType(tae, inputEncoder, baseRelation.output) .asInstanceOf[TypedAggregateExpression] @@ -2927,18 +2686,18 @@ class SparkConnectPlanner( case proto.Command.CommandTypeCase.STREAMING_QUERY_MANAGER_COMMAND => handleStreamingQueryManagerCommand( command.getStreamingQueryManagerCommand, - responseObserver - ) + responseObserver) case proto.Command.CommandTypeCase.STREAMING_QUERY_LISTENER_BUS_COMMAND => val handler = new SparkConnectStreamingQueryListenerHandler(executeHolder) - handler.handleListenerCommand(command.getStreamingQueryListenerBusCommand, responseObserver) + handler.handleListenerCommand( + command.getStreamingQueryListenerBusCommand, + responseObserver) case proto.Command.CommandTypeCase.GET_RESOURCES_COMMAND => handleGetResourcesCommand(responseObserver) case proto.Command.CommandTypeCase.CREATE_RESOURCE_PROFILE_COMMAND => handleCreateResourceProfileCommand( command.getCreateResourceProfileCommand, - responseObserver - ) + responseObserver) case proto.Command.CommandTypeCase.CHECKPOINT_COMMAND => handleCheckpointCommand(command.getCheckpointCommand, responseObserver) case proto.Command.CommandTypeCase.REMOVE_CACHED_REMOTE_RELATION_COMMAND => @@ -2947,8 +2706,6 @@ class SparkConnectPlanner( handleMergeIntoTableCommand(command.getMergeIntoTableCommand) case proto.Command.CommandTypeCase.ML_COMMAND => handleMlCommand(command.getMlCommand, responseObserver) - case proto.Command.CommandTypeCase.PIPELINE_COMMAND => - handlePipelineCommand(command.getPipelineCommand, responseObserver) case proto.Command.CommandTypeCase.EXECUTE_EXTERNAL_COMMAND => handleExecuteExternalCommand(command.getExecuteExternalCommand, responseObserver) @@ -2967,14 +2724,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setMlCommandResult(result) - .build() - ) - } - - private def handlePipelineCommand( - command: proto.PipelineCommand, - responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { - throw InvalidPlanInput(s"PipelineCommand is not supported in Spark Connect.") + .build()) } private def handleExecuteExternalCommand( @@ -2990,8 +2740,7 @@ class SparkConnectPlanner( session, command.getRunner, command.getCommand, - command.getOptionsMap.asScala.toMap - ) + command.getOptionsMap.asScala.toMap) val result = executor.execute() executeHolder.eventsManager.postFinished(Some(result.size)) @@ -3001,8 +2750,7 @@ class SparkConnectPlanner( StringEncoder, ArrowUtils.rootAllocator, session.sessionState.conf.sessionLocalTimeZone, - session.sessionState.conf.arrowUseLargeVarTypes - ) + session.sessionState.conf.arrowUseLargeVarTypes) val sqlCommandResult = SqlCommandResult.newBuilder() sqlCommandResult.getRelationBuilder.getLocalRelationBuilder.setData(arrowData) responseObserver.onNext( @@ -3011,8 +2759,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setSqlCommandResult(sqlCommandResult) - .build() - ) + .build()) } private def handleSqlCommand( @@ -3034,8 +2781,7 @@ class SparkConnectPlanner( .putAllNamedArguments(command.getNamedArgumentsMap) .addAllPosArgs(command.getPosArgsList) .addAllPosArguments(command.getPosArgumentsList) - .build() - ) + .build()) .build() } @@ -3046,8 +2792,7 @@ class SparkConnectPlanner( executeSQLWithRefs(relation.getWithRelations, tracker) case other => throw InvalidPlanInput( - s"SQL command expects either a SQL or a WithRelations, but got $other" - ) + s"SQL command expects either a SQL or a WithRelations, but got $other") } // Check if commands have been executed. @@ -3076,8 +2821,7 @@ class SparkConnectPlanner( schema, timeZoneId, errorOnDuplicatedFieldNames = false, - largeVarTypes = largeVarTypes - ) + largeVarTypes = largeVarTypes) } else { val batches = ArrowConverters.toBatchWithSchemaIterator( rowIter = rows.iterator, @@ -3086,8 +2830,7 @@ class SparkConnectPlanner( maxEstimatedBatchSize = maxBatchSize, timeZoneId = timeZoneId, errorOnDuplicatedFieldNames = false, - largeVarTypes = largeVarTypes - ) + largeVarTypes = largeVarTypes) assert(batches.hasNext) val bytes = batches.next() assert(!batches.hasNext, s"remaining batches: ${batches.size}") @@ -3100,9 +2843,7 @@ class SparkConnectPlanner( .setLocalRelation( proto.LocalRelation .newBuilder() - .setData(ByteString.copyFrom(bytes)) - ) - ) + .setData(ByteString.copyFrom(bytes)))) metrics.addAllMetrics(MetricGenerator.transformPlan(df).asJava) } else { // No execution triggered for relations. Manually set ready @@ -3117,8 +2858,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setSqlCommandResult(result) - .build() - ) + .build()) // Send Metrics when isCommand (i.e. show tables) which is eagerly executed & has metrics // Skip metrics when !isCommand (i.e. select 1) which is not executed & doesn't have metrics @@ -3129,8 +2869,7 @@ class SparkConnectPlanner( .setSessionId(sessionHolder.sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setMetrics(metrics.build) - .build - ) + .build) } } @@ -3189,20 +2928,17 @@ class SparkConnectPlanner( session.sql( sql.getQuery, namedArguments.asScala.toMap.transform((_, e) => Column(transformExpression(e))), - tracker - ) + tracker) } else if (!posArguments.isEmpty) { session.sql( sql.getQuery, posArguments.asScala.map(e => Column(transformExpression(e))).toArray, - tracker - ) + tracker) } else if (!args.isEmpty) { session.sql( sql.getQuery, args.asScala.toMap.transform((_, v) => transformLiteral(v)), - tracker - ) + tracker) } else if (!posArgs.isEmpty) { session.sql(sql.getQuery, posArgs.asScala.map(transformLiteral).toArray, tracker) } else { @@ -3221,8 +2957,7 @@ class SparkConnectPlanner( handleRegisterScalaUDF(fun) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") } executeHolder.eventsManager.postFinished() } @@ -3235,8 +2970,7 @@ class SparkConnectPlanner( session.udtf.registerPython(fun.getFunctionName, function) case _ => throw InvalidPlanInput( - s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported" - ) + s"Function with ID: ${fun.getFunctionCase.getNumber} is not supported") } executeHolder.eventsManager.postFinished() } @@ -3250,8 +2984,7 @@ class SparkConnectPlanner( session.dataSource.registerPython(fun.getName, dataSource) case _ => throw InvalidPlanInput( - s"Data source with ID: ${fun.getDataSourceCase.getNumber} is not supported" - ) + s"Data source with ID: ${fun.getDataSourceCase.getNumber} is not supported") } executeHolder.eventsManager.postFinished() } @@ -3265,8 +2998,7 @@ class SparkConnectPlanner( case dt => throw InvalidPlanInput( "Invalid Python user-defined table function return type. " + - s"Expect a struct type, but got ${dt.typeName}." - ) + s"Expect a struct type, but got ${dt.typeName}.") } } else { None @@ -3277,8 +3009,7 @@ class SparkConnectPlanner( func = transformPythonTableFunction(udtf), returnType = returnType, pythonEvalType = udtf.getEvalType, - udfDeterministic = fun.getDeterministic - ) + udfDeterministic = fun.getDeterministic) } private def handleRegisterPythonUDF(fun: proto.CommonInlineUserDefinedFunction): Unit = { @@ -3307,9 +3038,9 @@ class SparkConnectPlanner( private def handleCommandPlugin(extension: ProtoAny): Unit = { SparkConnectPluginRegistry.commandRegistry - // Lazily traverse the collection. - .view - // Apply the transformation. + // Lazily traverse the collection. + .view + // Apply the transformation. .map(p => p.process(extension.toByteArray, this)) // Find the first non-empty transformation or throw. .find(_ == true) @@ -3338,8 +3069,7 @@ class SparkConnectPlanner( plan = transformRelation(createView.getInput), allowExisting = false, replace = createView.getReplace, - viewType = viewType - ) + viewType = viewType) val tracker = executeHolder.eventsManager.createQueryPlanningTracker() Dataset.ofRows(session, plan, tracker).queryExecution.commandExecuted @@ -3381,8 +3111,7 @@ class SparkConnectPlanner( val cols = op.getBucketColumnNamesList.asScala if (op.getNumBuckets <= 0) { throw InvalidCommandInput( - s"BucketBy must specify a bucket count > 0, received ${op.getNumBuckets} instead." - ) + s"BucketBy must specify a bucket count > 0, received ${op.getNumBuckets} instead.") } w.bucketBy(op.getNumBuckets, cols.head, cols.tail.toSeq: _*) } @@ -3414,14 +3143,12 @@ class SparkConnectPlanner( case _ => throw new UnsupportedOperationException( "WriteOperation:SaveTable:TableSaveMethod not supported " - + s"${writeOperation.getTable.getSaveMethodValue}" - ) + + s"${writeOperation.getTable.getSaveMethodValue}") } case _ => throw new UnsupportedOperationException( "WriteOperation:SaveTypeCase not supported " - + s"${writeOperation.getSaveTypeCase.getNumber}" - ) + + s"${writeOperation.getSaveTypeCase.getNumber}") } executeHolder.eventsManager.postFinished() } @@ -3449,9 +3176,8 @@ class SparkConnectPlanner( } if (writeOperation.getTablePropertiesCount > 0) { - writeOperation.getTablePropertiesMap.asScala.foreach { - case (key, value) => - w.tableProperty(key, value) + writeOperation.getTablePropertiesMap.asScala.foreach { case (key, value) => + w.tableProperty(key, value) } } @@ -3495,8 +3221,7 @@ class SparkConnectPlanner( } case _ => throw new UnsupportedOperationException( - s"WriteOperationV2:ModeValue not supported ${writeOperation.getModeValue}" - ) + s"WriteOperationV2:ModeValue not supported ${writeOperation.getModeValue}") } executeHolder.eventsManager.postFinished() } @@ -3551,14 +3276,13 @@ class SparkConnectPlanner( val foreach = writeOp.getForeachWriter.getPythonFunction val pythonFcn = transformPythonFunction(foreach) writer.foreachImplementation( - new PythonForeachWriter(pythonFcn, dataset.schema).asInstanceOf[ForeachWriter[Any]] - ) + new PythonForeachWriter(pythonFcn, dataset.schema).asInstanceOf[ForeachWriter[Any]]) } else { val foreachWriterPkt = unpackForeachWriter(writeOp.getForeachWriter.getScalaFunction) val clientWriter = foreachWriterPkt.foreachWriter val encoder: Option[ExpressionEncoder[Any]] = Try( - ExpressionEncoder(foreachWriterPkt.datasetEncoder.asInstanceOf[AgnosticEncoder[Any]]) - ).toOption + ExpressionEncoder( + foreachWriterPkt.datasetEncoder.asInstanceOf[AgnosticEncoder[Any]])).toOption writer.foreachImplementation(clientWriter.asInstanceOf[ForeachWriter[Any]], encoder) } } @@ -3578,8 +3302,7 @@ class SparkConnectPlanner( case StreamingForeachFunction.FunctionCase.SCALA_FUNCTION => StreamingForeachBatchHelper.scalaForeachBatchWrapper( writeOp.getForeachBatch.getScalaFunction.getPayload.toByteArray, - sessionHolder - ) + sessionHolder) case StreamingForeachFunction.FunctionCase.FUNCTION_NOT_SET => throw InvalidPlanInput("Unexpected foreachBatch function") // Unreachable @@ -3599,8 +3322,7 @@ class SparkConnectPlanner( case NonFatal(ex) => // Failed to start the query, clean up foreach runner if any. logInfo( log"Removing foreachBatch worker, query failed to start " + - log"for session ${MDC(SESSION_ID, sessionId)}." - ) + log"for session ${MDC(SESSION_ID, sessionId)}.") foreachBatchRunnerCleaner.foreach(_.close()) throw ex } @@ -3610,11 +3332,12 @@ class SparkConnectPlanner( sessionHolder, query, executeHolder.sparkSessionTags, - executeHolder.operationId - ) + executeHolder.operationId) // Register the runner with the query if Python foreachBatch is enabled. foreachBatchRunnerCleaner.foreach { cleaner => - sessionHolder.streamingForeachBatchRunnerCleanerCache.registerCleanerForQuery(query, cleaner) + sessionHolder.streamingForeachBatchRunnerCleanerCache.registerCleanerForQuery( + query, + cleaner) } executeHolder.eventsManager.postFinished() @@ -3625,8 +3348,7 @@ class SparkConnectPlanner( .newBuilder() .setId(query.id.toString) .setRunId(query.runId.toString) - .build() - ) + .build()) .setName(Option(query.name).getOrElse("")) // The query started event for this query is sent to the client, and is handled by @@ -3643,18 +3365,16 @@ class SparkConnectPlanner( // before the client side DataStreamWriter.start(). // This way we ensure that the onQueryStarted API is called before the start() call in Connect. val queryStartedEvent = Option( - sessionHolder.streamingServersideListenerHolder.streamingQueryStartedEventCache - .remove(query.runId.toString) - ) + sessionHolder.streamingServersideListenerHolder.streamingQueryStartedEventCache.remove( + query.runId.toString)) queryStartedEvent.foreach { logInfo( log"[SessionId: ${MDC(LogKeys.SESSION_ID, sessionId)}]" + - log"[UserId: ${MDC(LogKeys.USER_ID, userId)}] " + - log"[operationId: ${MDC(LogKeys.OPERATION_ID, executeHolder.operationId)}] " + - log"[query id: ${MDC(LogKeys.QUERY_ID, query.id)}]" + - log"[query runId: ${MDC(LogKeys.QUERY_RUN_ID, query.runId)}] " + - log"Adding QueryStartedEvent to response" - ) + log"[UserId: ${MDC(LogKeys.USER_ID, userId)}] " + + log"[operationId: ${MDC(LogKeys.OPERATION_ID, executeHolder.operationId)}] " + + log"[query id: ${MDC(LogKeys.QUERY_ID, query.id)}]" + + log"[query runId: ${MDC(LogKeys.QUERY_RUN_ID, query.runId)}] " + + log"Adding QueryStartedEvent to response") e => resultBuilder.setQueryStartedEventJson(e.json) } @@ -3664,8 +3384,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setWriteStreamOperationStartResult(resultBuilder.build()) - .build() - ) + .build()) } private def handleStreamingQueryCommand( @@ -3681,7 +3400,7 @@ class SparkConnectPlanner( // Find the query in connect service level cache, otherwise check session's active streams. val query = SparkConnectService.streamingSessionManager - // Common case: query is cached in the cache. + // Common case: query is cached in the cache. .getCachedQuery(id, runId, executeHolder.sparkSessionTags, session) .map(_.query) .orElse { // Else try to find it in active streams. Mostly will not be found here either. @@ -3692,8 +3411,7 @@ class SparkConnectPlanner( case Some(query) => throw new IllegalArgumentException( s"Run id mismatch for query id $id. Run id in the request $runId " + - s"does not match one on the server ${query.runId}. The query might have restarted." - ) + s"does not match one on the server ${query.runId}. The query might have restarted.") case None => throw new IllegalArgumentException(s"Streaming query $id is not found") } @@ -3722,9 +3440,9 @@ class SparkConnectPlanner( respBuilder.setRecentProgress( StreamingQueryCommandResult.RecentProgressResult .newBuilder() - .addAllRecentProgressJson(progressReports.map(StreamingQueryProgress.jsonString).asJava) - .build() - ) + .addAllRecentProgressJson( + progressReports.map(StreamingQueryProgress.jsonString).asJava) + .build()) case StreamingQueryCommand.CommandCase.STOP => query.stop() @@ -3789,8 +3507,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setStreamingQueryCommandResult(respBuilder.build()) - .build() - ) + .build()) } /** @@ -3843,8 +3560,7 @@ class SparkConnectPlanner( .newBuilder() .setId(query.id.toString) .setRunId(query.runId.toString) - .build() - ) + .build()) if (query.name != null) { builder.setName(query.name) } @@ -3863,8 +3579,7 @@ class SparkConnectPlanner( active_queries .map(query => buildStreamingQueryInstance(query)) .toImmutableArraySeq - .asJava - ) + .asJava) case StreamingQueryManagerCommand.CommandCase.GET_QUERY => Option(session.streams.get(command.getGetQuery)).foreach { q => @@ -3889,14 +3604,12 @@ class SparkConnectPlanner( val listener = if (command.getAddListener.hasPythonListenerPayload) { new PythonStreamingQueryListener( transformPythonFunction(command.getAddListener.getPythonListenerPayload), - sessionHolder - ) + sessionHolder) } else { val listenerPacket = Utils .deserialize[StreamingListenerPacket]( command.getAddListener.getListenerPayload.toByteArray, - Utils.getContextOrSparkClassLoader - ) + Utils.getContextOrSparkClassLoader) listenerPacket.listener.asInstanceOf[StreamingQueryListener] } @@ -3932,8 +3645,7 @@ class SparkConnectPlanner( .setSessionId(sessionId) .setServerSideSessionId(sessionHolder.serverSessionId) .setStreamingQueryManagerCommandResult(respBuilder.build()) - .build() - ) + .build()) } private def handleGetResourcesCommand( @@ -3949,38 +3661,30 @@ class SparkConnectPlanner( .newBuilder() .putAllResources( session.sparkContext.resources.toMap - .transform( - (_, resource) => - proto.ResourceInformation - .newBuilder() - .setName(resource.name) - .addAllAddresses(resource.addresses.toImmutableArraySeq.asJava) - .build() - ) - .asJava - ) - .build() - ) - .build() - ) + .transform((_, resource) => + proto.ResourceInformation + .newBuilder() + .setName(resource.name) + .addAllAddresses(resource.addresses.toImmutableArraySeq.asJava) + .build()) + .asJava) + .build()) + .build()) } private def handleCreateResourceProfileCommand( createResourceProfileCommand: CreateResourceProfileCommand, responseObserver: StreamObserver[proto.ExecutePlanResponse]): Unit = { val rp = createResourceProfileCommand.getProfile - val ereqs = rp.getExecutorResourcesMap.asScala.map { - case (name, res) => - name -> new ExecutorResourceRequest( - res.getResourceName, - res.getAmount, - res.getDiscoveryScript, - res.getVendor - ) + val ereqs = rp.getExecutorResourcesMap.asScala.map { case (name, res) => + name -> new ExecutorResourceRequest( + res.getResourceName, + res.getAmount, + res.getDiscoveryScript, + res.getVendor) }.toMap - val treqs = rp.getTaskResourcesMap.asScala.map { - case (name, res) => - name -> new TaskResourceRequest(res.getResourceName, res.getAmount) + val treqs = rp.getTaskResourcesMap.asScala.map { case (name, res) => + name -> new TaskResourceRequest(res.getResourceName, res.getAmount) }.toMap // Create ResourceProfile add add it to ResourceProfileManager @@ -4001,10 +3705,8 @@ class SparkConnectPlanner( proto.CreateResourceProfileCommandResult .newBuilder() .setProfileId(profile.id) - .build() - ) - .build() - ) + .build()) + .build()) } private def handleCheckpointCommand( @@ -4017,8 +3719,7 @@ class SparkConnectPlanner( target.localCheckpoint( eager = checkpointCommand.getEager, storageLevel = - StorageLevelProtoConverter.toStorageLevel(checkpointCommand.getStorageLevel) - ) + StorageLevelProtoConverter.toStorageLevel(checkpointCommand.getStorageLevel)) } else { target.localCheckpoint(eager = checkpointCommand.getEager) } @@ -4040,10 +3741,8 @@ class SparkConnectPlanner( proto.CheckpointCommandResult .newBuilder() .setRelation(proto.CachedRemoteRelation.newBuilder().setRelationId(dfId).build()) - .build() - ) - .build() - ) + .build()) + .build()) } private def handleRemoveCachedRemoteRelationCommand( @@ -4078,8 +3777,7 @@ class SparkConnectPlanner( private val emptyLocalRelation = LocalRelation( output = AttributeReference("value", StringType, false)() :: Nil, - data = Seq.empty - ) + data = Seq.empty) private def transformCurrentDatabase(): LogicalPlan = { session.createDataset(session.catalog.currentDatabase :: Nil)(Encoders.STRING).logicalPlan @@ -4151,10 +3849,10 @@ class SparkConnectPlanner( if (getGetTable.hasDbName) { Catalog .makeDataset( - session.catalog - .getTable(dbName = getGetTable.getDbName, tableName = getGetTable.getTableName) :: Nil, - session - ) + session.catalog.getTable( + dbName = getGetTable.getDbName, + tableName = getGetTable.getTableName) :: Nil, + session) .logicalPlan } else { Catalog @@ -4169,10 +3867,8 @@ class SparkConnectPlanner( .makeDataset( session.catalog.getFunction( dbName = getGetFunction.getDbName, - functionName = getGetFunction.getFunctionName - ) :: Nil, - session - ) + functionName = getGetFunction.getFunctionName) :: Nil, + session) .logicalPlan } else { Catalog @@ -4184,8 +3880,7 @@ class SparkConnectPlanner( private def transformDatabaseExists(getDatabaseExists: proto.DatabaseExists): LogicalPlan = { session .createDataset(session.catalog.databaseExists(getDatabaseExists.getDbName) :: Nil)( - Encoders.scalaBoolean - ) + Encoders.scalaBoolean) .logicalPlan } @@ -4195,15 +3890,12 @@ class SparkConnectPlanner( .createDataset( session.catalog.tableExists( dbName = getTableExists.getDbName, - tableName = getTableExists.getTableName - ) :: Nil - )(Encoders.scalaBoolean) + tableName = getTableExists.getTableName) :: Nil)(Encoders.scalaBoolean) .logicalPlan } else { session .createDataset(session.catalog.tableExists(getTableExists.getTableName) :: Nil)( - Encoders.scalaBoolean - ) + Encoders.scalaBoolean) .logicalPlan } } @@ -4214,15 +3906,12 @@ class SparkConnectPlanner( .createDataset( session.catalog.functionExists( dbName = getFunctionExists.getDbName, - functionName = getFunctionExists.getFunctionName - ) :: Nil - )(Encoders.scalaBoolean) + functionName = getFunctionExists.getFunctionName) :: Nil)(Encoders.scalaBoolean) .logicalPlan } else { session .createDataset(session.catalog.functionExists(getFunctionExists.getFunctionName) :: Nil)( - Encoders.scalaBoolean - ) + Encoders.scalaBoolean) .logicalPlan } } @@ -4245,7 +3934,7 @@ class SparkConnectPlanner( val options = if (getCreateExternalTable.hasPath) { (getCreateExternalTable.getOptionsMap.asScala ++ - Map("path" -> getCreateExternalTable.getPath)).asJava + Map("path" -> getCreateExternalTable.getPath)).asJava } else { getCreateExternalTable.getOptionsMap } @@ -4254,8 +3943,7 @@ class SparkConnectPlanner( tableName = getCreateExternalTable.getTableName, source = source, schema = schema, - options = options - ) + options = options) .logicalPlan } @@ -4282,7 +3970,7 @@ class SparkConnectPlanner( val options = if (getCreateTable.hasPath) { (getCreateTable.getOptionsMap.asScala ++ - Map("path" -> getCreateTable.getPath)).asJava + Map("path" -> getCreateTable.getPath)).asJava } else { getCreateTable.getOptionsMap } @@ -4293,25 +3981,23 @@ class SparkConnectPlanner( source = source, schema = schema, description = description, - options = options - ) + options = options) .logicalPlan } private def transformDropTempView(getDropTempView: proto.DropTempView): LogicalPlan = { session .createDataset(session.catalog.dropTempView(getDropTempView.getViewName) :: Nil)( - Encoders.scalaBoolean - ) + Encoders.scalaBoolean) .logicalPlan } private def transformDropGlobalTempView( getDropGlobalTempView: proto.DropGlobalTempView): LogicalPlan = { session - .createDataset(session.catalog.dropGlobalTempView(getDropGlobalTempView.getViewName) :: Nil)( - Encoders.scalaBoolean - ) + .createDataset( + session.catalog.dropGlobalTempView(getDropGlobalTempView.getViewName) :: Nil)( + Encoders.scalaBoolean) .logicalPlan } @@ -4324,8 +4010,7 @@ class SparkConnectPlanner( private def transformIsCached(getIsCached: proto.IsCached): LogicalPlan = { session .createDataset(session.catalog.isCached(getIsCached.getTableName) :: Nil)( - Encoders.scalaBoolean - ) + Encoders.scalaBoolean) .logicalPlan } @@ -4333,8 +4018,7 @@ class SparkConnectPlanner( if (getCacheTable.hasStorageLevel) { session.catalog.cacheTable( getCacheTable.getTableName, - StorageLevelProtoConverter.toStorageLevel(getCacheTable.getStorageLevel) - ) + StorageLevelProtoConverter.toStorageLevel(getCacheTable.getStorageLevel)) } else { session.catalog.cacheTable(getCacheTable.getTableName) } @@ -4399,16 +4083,16 @@ class SparkConnectPlanner( .map(transformSortOrder) .toSeq, withSinglePartition = - options.hasWithSinglePartition && options.getWithSinglePartition - ) + options.hasWithSinglePartition && options.getWithSinglePartition) } else { UnresolvedTableArgPlanId(planId) } case proto.SubqueryExpression.SubqueryType.SUBQUERY_TYPE_IN => - UnresolvedInSubqueryPlanId(getSubqueryExpression.getInSubqueryValuesList.asScala.map { - value => + UnresolvedInSubqueryPlanId( + getSubqueryExpression.getInSubqueryValuesList.asScala.map { value => transformExpression(value) - }.toSeq, planId) + }.toSeq, + planId) case other => throw InvalidPlanInput(s"Unknown SubqueryType $other") } } @@ -4444,9 +4128,8 @@ class SparkConnectPlanner( assertPlan( missingPlanIds.isEmpty, "Missing relation in WithRelations: " + - s"${missingPlanIds.mkString("(", ", ", ")")} not in " + - s"${relations.keys.mkString("(", ", ", ")")}" - ) + s"${missingPlanIds.mkString("(", ", ", ")")} not in " + + s"${relations.keys.mkString("(", ", ", ")")}") withRelations } } From 2b2f6373eb9c37a76ef6babf6aa16e43e893e3f7 Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Tue, 20 May 2025 13:03:07 -0700 Subject: [PATCH 3/9] 1 --- .../protobuf/spark/connect/pipelines.proto | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index cd65168d37a46..cf0f74dd32a47 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -8,7 +8,7 @@ import "spark/connect/types.proto"; option java_multiple_files = true; option java_package = "org.apache.spark.connect.proto"; -// Dispatch object for pipelines commands. +// Dispatch object for pipelines commands. See each individual command for documentation. message PipelineCommand { oneof command_type { CreateDataflowGraph create_dataflow_graph = 1; @@ -20,13 +20,7 @@ message PipelineCommand { DefineSqlGraphElements define_sql_graph_elements = 7; } - message DefineSqlGraphElements { - optional string dataflow_graph_id = 1; - optional string sql_file_name = 2; - optional string sql_text = 3; - } - - // Request to create a new pipeline. + // Request to create a new dataflow graph. message CreateDataflowGraph { // The default catalog. optional string default_catalog = 1; @@ -34,7 +28,7 @@ message PipelineCommand { // The default database. optional string default_database = 2; - // Default SQL configurations for all flows in this graph. + // SQL configurations for all flows in this graph. map sql_conf = 5; message Response { @@ -54,25 +48,27 @@ message PipelineCommand { // The graph to attach this dataset to. string dataflow_graph_id = 1; - // Name of the dataset. + // Name of the dataset. Can be partially or fully qualified. string dataset_name = 2; - // Table or view. + // The type of the dataset. DatasetType dataset_type = 3; // Optional comment for the dataset. optional string comment = 4; - // Optional table properties. + // Optional table properties. Only applies to dataset_type == TABLE and dataset_type == MATERIALIZED_VIEW. map table_properties = 5; - // Optional partition columns for the dataset (if applicable). + // Optional partition columns for the dataset. Only applies to dataset_type == TABLE and + // dataset_type == MATERIALIZED_VIEW. repeated string partition_cols = 6; - // Schema for the dataset. If unset, this will be inferred. + // Schema for the dataset. If unset, this will be inferred from incoming flows. optional spark.connect.DataType schema = 7; - // The output table format of the dataset. + // The output table format of the dataset. Only applies to dataset_type == TABLE and + // dataset_type == MATERIALIZED_VIEW. optional string format = 8; } @@ -81,9 +77,10 @@ message PipelineCommand { // The graph to attach this dataset to. string dataflow_graph_id = 1; - // Name of the dataset. + // Name of the flow. For standalone flows, this must be a single-part name. string flow_name = 2; + // Name of the dataset this flow writes to. Can be partially or fully qualified. string target_dataset_name = 3; // An unresolved relation that defines the dataset's flow. @@ -102,12 +99,26 @@ message PipelineCommand { // The graph to start. string dataflow_graph_id = 1; } + + // Stops all running flows in the graph. This is a no-op if the graph is not running. message StopRun { - // The ID of the graph to stop.. + // The ID of the graph to stop. string dataflow_graph_id = 1; } } +// Parses the SQL file and registers all datasets and flows. +message DefineSqlGraphElements { + // The graph to attach this dataset to. + optional string dataflow_graph_id = 1; + + // The full path to the SQL file. Can be relative or absolute. + optional string sql_file_path = 2; + + // The contents of the SQL file. + optional string sql_text = 3; +} + // Dispatch object for pipelines command results. message PipelineCommandResult { oneof result_type { @@ -119,7 +130,9 @@ message PipelineCommandResult { } } +// The type of dataset. enum DatasetType { + // Safe default value. Should not be used. DATASET_UNSPECIFIED = 0; // A materialized view dataset which is published to the catalog MATERIALIZED_VIEW = 1; @@ -134,6 +147,7 @@ message PipelineEventsResult { repeated PipelineEvent events = 1; } +// An event emitted during the run of a graph. message PipelineEvent { // The time of the event. optional string timestamp = 1; From ad5b0396eb84194023259531ba1d02703e16bfab Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Tue, 20 May 2025 13:03:36 -0700 Subject: [PATCH 4/9] license --- .../main/protobuf/spark/connect/pipelines.proto | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index cf0f74dd32a47..27ffa976ddab7 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; package spark.connect; From 988a3047c51e691bcbc85882ecd4952d7a6db480 Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Wed, 21 May 2025 12:18:03 -0700 Subject: [PATCH 5/9] 2 --- .../src/main/protobuf/spark/connect/pipelines.proto | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index 27ffa976ddab7..9c90a5bcded7b 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -33,8 +33,7 @@ message PipelineCommand { DefineFlow define_flow = 3; DropDataflowGraph drop_dataflow_graph = 4; StartRun start_run = 5; - StopRun stop_run = 6; - DefineSqlGraphElements define_sql_graph_elements = 7; + DefineSqlGraphElements define_sql_graph_elements = 6; } // Request to create a new dataflow graph. @@ -116,12 +115,6 @@ message PipelineCommand { // The graph to start. string dataflow_graph_id = 1; } - - // Stops all running flows in the graph. This is a no-op if the graph is not running. - message StopRun { - // The ID of the graph to stop. - string dataflow_graph_id = 1; - } } // Parses the SQL file and registers all datasets and flows. From f855349cb37189f2eee0b4d9487eb260f8bb1ff7 Mon Sep 17 00:00:00 2001 From: Aakash Japi Date: Wed, 21 May 2025 12:27:13 -0700 Subject: [PATCH 6/9] comments --- .../protobuf/spark/connect/pipelines.proto | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index 9c90a5bcded7b..32f69c32a32ac 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -49,26 +49,26 @@ message PipelineCommand { message Response { // The ID of the created graph. - string dataflow_graph_id = 1; + optional string dataflow_graph_id = 1; } } // Drops the graph and stops any running attached flows. message DropDataflowGraph { // The graph to drop. - string dataflow_graph_id = 1; + optional string dataflow_graph_id = 1; } // Request to define a dataset: a table, a materialized view, or a temporary view. message DefineDataset { // The graph to attach this dataset to. - string dataflow_graph_id = 1; + optional string dataflow_graph_id = 1; // Name of the dataset. Can be partially or fully qualified. - string dataset_name = 2; + optional string dataset_name = 2; // The type of the dataset. - DatasetType dataset_type = 3; + optional DatasetType dataset_type = 3; // Optional comment for the dataset. optional string comment = 4; @@ -90,30 +90,30 @@ message PipelineCommand { // Request to define a flow targeting a dataset. message DefineFlow { - // The graph to attach this dataset to. - string dataflow_graph_id = 1; + // The graph to attach this flow to. + optional string dataflow_graph_id = 1; // Name of the flow. For standalone flows, this must be a single-part name. - string flow_name = 2; + optional string flow_name = 2; // Name of the dataset this flow writes to. Can be partially or fully qualified. - string target_dataset_name = 3; + optional string target_dataset_name = 3; // An unresolved relation that defines the dataset's flow. - spark.connect.Relation plan = 4; + optional spark.connect.Relation plan = 4; - // Default SQL configurations set when running this flow. + // SQL configurations set when running this flow. map sql_conf = 5; // If true, this flow will only be run once per execution. - bool once = 6; + optional bool once = 6; } // Resolves all datasets and flows and start a pipeline update. Should be called after all // graph elements are registered. message StartRun { // The graph to start. - string dataflow_graph_id = 1; + optional string dataflow_graph_id = 1; } } @@ -136,7 +136,7 @@ message PipelineCommandResult { } message CreateDataflowGraphResult { // The ID of the created graph. - string dataflow_graph_id = 1; + optional string dataflow_graph_id = 1; } } From 5042b88122ce927e8dd2a3290f5ff046734cbfd7 Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Tue, 27 May 2025 08:48:17 -0700 Subject: [PATCH 7/9] fix once docstring --- .../common/src/main/protobuf/spark/connect/pipelines.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index 32f69c32a32ac..046755971e903 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -105,7 +105,7 @@ message PipelineCommand { // SQL configurations set when running this flow. map sql_conf = 5; - // If true, this flow will only be run once per execution. + // If true, this flow will only be run once per full refresh. optional bool once = 6; } From 4f8664877c5689afa8f2da49409234d7249355af Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Tue, 27 May 2025 08:51:17 -0700 Subject: [PATCH 8/9] take out event batching --- .../common/src/main/protobuf/spark/connect/base.proto | 5 ++--- .../src/main/protobuf/spark/connect/pipelines.proto | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index 422f30e46db0f..7f317defd47b5 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -400,9 +400,8 @@ message ExecutePlanResponse { // ML command response MlCommandResult ml_command_result = 20; - // Response containing pipeline events that are streamed back to the client during a pipeline - // run - PipelineEventsResult pipeline_events_result = 21; + // Response containing pipeline event that is streamed back to the client during a pipeline run + PipelineEventResult pipeline_event_result = 21; // Pipeline command response PipelineCommandResult pipeline_command_result = 22; diff --git a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto index 046755971e903..ea94f9b03bfc8 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto @@ -143,7 +143,7 @@ message PipelineCommandResult { // The type of dataset. enum DatasetType { // Safe default value. Should not be used. - DATASET_UNSPECIFIED = 0; + DATASET_TYPE_UNSPECIFIED = 0; // A materialized view dataset which is published to the catalog MATERIALIZED_VIEW = 1; // A table which is published to the catalog @@ -152,9 +152,9 @@ enum DatasetType { TEMPORARY_VIEW = 3; } -// A response containing events emitted during the run of a pipeline. -message PipelineEventsResult { - repeated PipelineEvent events = 1; +// A response containing an event emitted during the run of a pipeline. +message PipelineEventResult { + PipelineEvent event = 1; } // An event emitted during the run of a graph. From b1c2a4b48fdbe2b956bf2d89e95d41743a46ebbc Mon Sep 17 00:00:00 2001 From: Sandy Ryza Date: Tue, 27 May 2025 08:57:44 -0700 Subject: [PATCH 9/9] Generate python bindings --- python/pyspark/sql/connect/proto/base_pb2.py | 359 ++++---- python/pyspark/sql/connect/proto/base_pb2.pyi | 25 + .../pyspark/sql/connect/proto/commands_pb2.py | 211 ++--- .../sql/connect/proto/commands_pb2.pyi | 10 + .../sql/connect/proto/pipelines_pb2.py | 91 ++ .../sql/connect/proto/pipelines_pb2.pyi | 860 ++++++++++++++++++ 6 files changed, 1272 insertions(+), 284 deletions(-) create mode 100644 python/pyspark/sql/connect/proto/pipelines_pb2.py create mode 100644 python/pyspark/sql/connect/proto/pipelines_pb2.pyi diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index 2fbc4287db786..7d0c681576b90 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -41,10 +41,11 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2 from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2 from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2 +from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\xa3\x05\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\xa5\x01\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xb4\x17\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1av\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x42\x0f\n\r_start_offset\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x8d\x01\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x93\x0c\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\x99\x03\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_state\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation2\xb2\x07\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\xa3\x05\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\xa5\x01\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xee\x18\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12L\n\x11ml_command_result\x18\x14 \x01(\x0b\x32\x1e.spark.connect.MlCommandResultH\x00R\x0fmlCommandResult\x12X\n\x15pipeline_event_result\x18\x15 \x01(\x0b\x32".spark.connect.PipelineEventResultH\x00R\x13pipelineEventResult\x12^\n\x17pipeline_command_result\x18\x16 \x01(\x0b\x32$.spark.connect.PipelineCommandResultH\x00R\x15pipelineCommandResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1av\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x42\x0f\n\r_start_offset\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x8d\x01\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x93\x0c\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\x99\x03\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_state\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation2\xb2\x07\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -69,190 +70,190 @@ _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" ]._serialized_options = b"8\001" - _globals["_PLAN"]._serialized_start = 243 - _globals["_PLAN"]._serialized_end = 359 - _globals["_USERCONTEXT"]._serialized_start = 361 - _globals["_USERCONTEXT"]._serialized_end = 483 - _globals["_ANALYZEPLANREQUEST"]._serialized_start = 486 - _globals["_ANALYZEPLANREQUEST"]._serialized_end = 3163 - _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_start = 1848 - _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_end = 1897 - _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_start = 1900 - _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_end = 2215 - _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_start = 2043 - _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_end = 2215 - _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_start = 2217 - _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_end = 2307 - _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_start = 2309 - _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_end = 2359 - _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_start = 2361 - _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_end = 2415 - _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_start = 2417 - _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_end = 2470 - _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_start = 2472 - _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_end = 2486 - _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_start = 2488 - _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_end = 2529 - _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_start = 2531 - _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_end = 2652 - _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_start = 2654 - _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_end = 2709 - _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_start = 2712 - _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_end = 2863 - _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_start = 2865 - _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_end = 2975 - _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_start = 2977 - _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_end = 3047 - _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_start = 3049 - _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_end = 3093 - _globals["_ANALYZEPLANRESPONSE"]._serialized_start = 3166 - _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 5032 - _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_start = 4407 - _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_end = 4464 - _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_start = 4466 - _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_end = 4514 - _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_start = 4516 - _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_end = 4561 - _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_start = 4563 - _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_end = 4599 - _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_start = 4601 - _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_end = 4649 - _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_start = 4651 - _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_end = 4685 - _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_start = 4687 - _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 4727 - _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 4729 - _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 4788 - _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 4790 - _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 4829 - _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 4831 - _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 4869 - _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_start = 2712 - _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_end = 2721 - _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_start = 2865 - _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_end = 2876 - _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 4895 - _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 4978 - _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_start = 4980 - _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_end = 5022 - _globals["_EXECUTEPLANREQUEST"]._serialized_start = 5035 - _globals["_EXECUTEPLANREQUEST"]._serialized_end = 5710 - _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5469 - _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 5634 - _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 5713 - _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 8709 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 7483 - _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 7554 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 7556 - _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 7674 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 7677 - _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 8194 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 7772 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 8104 + _globals["_PLAN"]._serialized_start = 274 + _globals["_PLAN"]._serialized_end = 390 + _globals["_USERCONTEXT"]._serialized_start = 392 + _globals["_USERCONTEXT"]._serialized_end = 514 + _globals["_ANALYZEPLANREQUEST"]._serialized_start = 517 + _globals["_ANALYZEPLANREQUEST"]._serialized_end = 3194 + _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_start = 1879 + _globals["_ANALYZEPLANREQUEST_SCHEMA"]._serialized_end = 1928 + _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_start = 1931 + _globals["_ANALYZEPLANREQUEST_EXPLAIN"]._serialized_end = 2246 + _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_start = 2074 + _globals["_ANALYZEPLANREQUEST_EXPLAIN_EXPLAINMODE"]._serialized_end = 2246 + _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_start = 2248 + _globals["_ANALYZEPLANREQUEST_TREESTRING"]._serialized_end = 2338 + _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_start = 2340 + _globals["_ANALYZEPLANREQUEST_ISLOCAL"]._serialized_end = 2390 + _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_start = 2392 + _globals["_ANALYZEPLANREQUEST_ISSTREAMING"]._serialized_end = 2446 + _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_start = 2448 + _globals["_ANALYZEPLANREQUEST_INPUTFILES"]._serialized_end = 2501 + _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_start = 2503 + _globals["_ANALYZEPLANREQUEST_SPARKVERSION"]._serialized_end = 2517 + _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_start = 2519 + _globals["_ANALYZEPLANREQUEST_DDLPARSE"]._serialized_end = 2560 + _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_start = 2562 + _globals["_ANALYZEPLANREQUEST_SAMESEMANTICS"]._serialized_end = 2683 + _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_start = 2685 + _globals["_ANALYZEPLANREQUEST_SEMANTICHASH"]._serialized_end = 2740 + _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_start = 2743 + _globals["_ANALYZEPLANREQUEST_PERSIST"]._serialized_end = 2894 + _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_start = 2896 + _globals["_ANALYZEPLANREQUEST_UNPERSIST"]._serialized_end = 3006 + _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_start = 3008 + _globals["_ANALYZEPLANREQUEST_GETSTORAGELEVEL"]._serialized_end = 3078 + _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_start = 3080 + _globals["_ANALYZEPLANREQUEST_JSONTODDL"]._serialized_end = 3124 + _globals["_ANALYZEPLANRESPONSE"]._serialized_start = 3197 + _globals["_ANALYZEPLANRESPONSE"]._serialized_end = 5063 + _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_start = 4438 + _globals["_ANALYZEPLANRESPONSE_SCHEMA"]._serialized_end = 4495 + _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_start = 4497 + _globals["_ANALYZEPLANRESPONSE_EXPLAIN"]._serialized_end = 4545 + _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_start = 4547 + _globals["_ANALYZEPLANRESPONSE_TREESTRING"]._serialized_end = 4592 + _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_start = 4594 + _globals["_ANALYZEPLANRESPONSE_ISLOCAL"]._serialized_end = 4630 + _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_start = 4632 + _globals["_ANALYZEPLANRESPONSE_ISSTREAMING"]._serialized_end = 4680 + _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_start = 4682 + _globals["_ANALYZEPLANRESPONSE_INPUTFILES"]._serialized_end = 4716 + _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_start = 4718 + _globals["_ANALYZEPLANRESPONSE_SPARKVERSION"]._serialized_end = 4758 + _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_start = 4760 + _globals["_ANALYZEPLANRESPONSE_DDLPARSE"]._serialized_end = 4819 + _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_start = 4821 + _globals["_ANALYZEPLANRESPONSE_SAMESEMANTICS"]._serialized_end = 4860 + _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_start = 4862 + _globals["_ANALYZEPLANRESPONSE_SEMANTICHASH"]._serialized_end = 4900 + _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_start = 2743 + _globals["_ANALYZEPLANRESPONSE_PERSIST"]._serialized_end = 2752 + _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_start = 2896 + _globals["_ANALYZEPLANRESPONSE_UNPERSIST"]._serialized_end = 2907 + _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_start = 4926 + _globals["_ANALYZEPLANRESPONSE_GETSTORAGELEVEL"]._serialized_end = 5009 + _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_start = 5011 + _globals["_ANALYZEPLANRESPONSE_JSONTODDL"]._serialized_end = 5053 + _globals["_EXECUTEPLANREQUEST"]._serialized_start = 5066 + _globals["_EXECUTEPLANREQUEST"]._serialized_end = 5741 + _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_start = 5500 + _globals["_EXECUTEPLANREQUEST_REQUESTOPTION"]._serialized_end = 5665 + _globals["_EXECUTEPLANRESPONSE"]._serialized_start = 5744 + _globals["_EXECUTEPLANRESPONSE"]._serialized_end = 8926 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_start = 7700 + _globals["_EXECUTEPLANRESPONSE_SQLCOMMANDRESULT"]._serialized_end = 7771 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_start = 7773 + _globals["_EXECUTEPLANRESPONSE_ARROWBATCH"]._serialized_end = 7891 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_start = 7894 + _globals["_EXECUTEPLANRESPONSE_METRICS"]._serialized_end = 8411 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_start = 7989 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT"]._serialized_end = 8321 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_start = 7981 + ]._serialized_start = 8198 _globals[ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY" - ]._serialized_end = 8104 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 8106 - _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 8194 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 8197 - _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 8338 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 8340 - _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 8356 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 8359 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 8692 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 8515 - _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 8692 - _globals["_KEYVALUE"]._serialized_start = 8711 - _globals["_KEYVALUE"]._serialized_end = 8776 - _globals["_CONFIGREQUEST"]._serialized_start = 8779 - _globals["_CONFIGREQUEST"]._serialized_end = 9978 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 9087 - _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9585 - _globals["_CONFIGREQUEST_SET"]._serialized_start = 9587 - _globals["_CONFIGREQUEST_SET"]._serialized_end = 9679 - _globals["_CONFIGREQUEST_GET"]._serialized_start = 9681 - _globals["_CONFIGREQUEST_GET"]._serialized_end = 9706 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9708 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9771 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9773 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9804 - _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9806 - _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9854 - _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9856 - _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9883 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9885 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9919 - _globals["_CONFIGRESPONSE"]._serialized_start = 9981 - _globals["_CONFIGRESPONSE"]._serialized_end = 10156 - _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10159 - _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11161 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10634 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10687 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10689 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 10800 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10802 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10895 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 10898 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 11091 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11164 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11436 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11355 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11436 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11439 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11765 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11768 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 12120 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 11963 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 12078 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 12080 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 12120 - _globals["_INTERRUPTREQUEST"]._serialized_start = 12123 - _globals["_INTERRUPTREQUEST"]._serialized_end = 12726 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12526 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12654 - _globals["_INTERRUPTRESPONSE"]._serialized_start = 12729 - _globals["_INTERRUPTRESPONSE"]._serialized_end = 12873 - _globals["_REATTACHOPTIONS"]._serialized_start = 12875 - _globals["_REATTACHOPTIONS"]._serialized_end = 12928 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12931 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13337 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13340 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13925 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13794 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13806 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13808 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13855 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13928 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 14093 - _globals["_RELEASESESSIONREQUEST"]._serialized_start = 14096 - _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14308 - _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14310 - _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14418 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14421 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14753 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14756 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16311 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 14985 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 15159 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 15162 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15530 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15493 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15530 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15533 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 15942 + ]._serialized_end = 8321 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 8323 + _globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 8411 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 8414 + _globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_end = 8555 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_start = 8557 + _globals["_EXECUTEPLANRESPONSE_RESULTCOMPLETE"]._serialized_end = 8573 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_start = 8576 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS"]._serialized_end = 8909 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_start = 8732 + _globals["_EXECUTEPLANRESPONSE_EXECUTIONPROGRESS_STAGEINFO"]._serialized_end = 8909 + _globals["_KEYVALUE"]._serialized_start = 8928 + _globals["_KEYVALUE"]._serialized_end = 8993 + _globals["_CONFIGREQUEST"]._serialized_start = 8996 + _globals["_CONFIGREQUEST"]._serialized_end = 10195 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 9304 + _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9802 + _globals["_CONFIGREQUEST_SET"]._serialized_start = 9804 + _globals["_CONFIGREQUEST_SET"]._serialized_end = 9896 + _globals["_CONFIGREQUEST_GET"]._serialized_start = 9898 + _globals["_CONFIGREQUEST_GET"]._serialized_end = 9923 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9925 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9988 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9990 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 10021 + _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 10023 + _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 10071 + _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 10073 + _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 10100 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 10102 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 10136 + _globals["_CONFIGRESPONSE"]._serialized_start = 10198 + _globals["_CONFIGRESPONSE"]._serialized_end = 10373 + _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10376 + _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11378 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10851 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10904 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10906 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 11017 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 11019 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 11112 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 11115 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 11308 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11381 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11653 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11572 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11653 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11656 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11982 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11985 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 12337 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 12180 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 12295 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 12297 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 12337 + _globals["_INTERRUPTREQUEST"]._serialized_start = 12340 + _globals["_INTERRUPTREQUEST"]._serialized_end = 12943 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12743 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12871 + _globals["_INTERRUPTRESPONSE"]._serialized_start = 12946 + _globals["_INTERRUPTRESPONSE"]._serialized_end = 13090 + _globals["_REATTACHOPTIONS"]._serialized_start = 13092 + _globals["_REATTACHOPTIONS"]._serialized_end = 13145 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 13148 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13554 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13557 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 14142 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 14011 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 14023 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 14025 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 14072 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 14145 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 14310 + _globals["_RELEASESESSIONREQUEST"]._serialized_start = 14313 + _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14525 + _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14527 + _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14635 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14638 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14970 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14973 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16528 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 15202 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 15376 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 15379 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15747 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15710 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15747 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15750 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 16159 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_start = 15844 + ]._serialized_start = 16061 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_end = 15912 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15945 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16292 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16313 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16403 - _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16406 - _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17352 + ]._serialized_end = 16129 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 16162 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16509 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16530 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16620 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16623 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17569 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index 738339fa968ec..a4bb0d2fcfd76 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -44,6 +44,7 @@ import pyspark.sql.connect.proto.commands_pb2 import pyspark.sql.connect.proto.common_pb2 import pyspark.sql.connect.proto.expressions_pb2 import pyspark.sql.connect.proto.ml_pb2 +import pyspark.sql.connect.proto.pipelines_pb2 import pyspark.sql.connect.proto.relations_pb2 import pyspark.sql.connect.proto.types_pb2 import sys @@ -1583,6 +1584,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): EXECUTION_PROGRESS_FIELD_NUMBER: builtins.int CHECKPOINT_COMMAND_RESULT_FIELD_NUMBER: builtins.int ML_COMMAND_RESULT_FIELD_NUMBER: builtins.int + PIPELINE_EVENT_RESULT_FIELD_NUMBER: builtins.int + PIPELINE_COMMAND_RESULT_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int METRICS_FIELD_NUMBER: builtins.int OBSERVED_METRICS_FIELD_NUMBER: builtins.int @@ -1650,6 +1653,14 @@ class ExecutePlanResponse(google.protobuf.message.Message): def ml_command_result(self) -> pyspark.sql.connect.proto.ml_pb2.MlCommandResult: """ML command response""" @property + def pipeline_event_result(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult: + """Response containing pipeline event that is streamed back to the client during a pipeline run""" + @property + def pipeline_command_result( + self, + ) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult: + """Pipeline command response""" + @property def extension(self) -> google.protobuf.any_pb2.Any: """Support arbitrary result objects.""" @property @@ -1692,6 +1703,10 @@ class ExecutePlanResponse(google.protobuf.message.Message): execution_progress: global___ExecutePlanResponse.ExecutionProgress | None = ..., checkpoint_command_result: global___CheckpointCommandResult | None = ..., ml_command_result: pyspark.sql.connect.proto.ml_pb2.MlCommandResult | None = ..., + pipeline_event_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult + | None = ..., + pipeline_command_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult + | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., metrics: global___ExecutePlanResponse.Metrics | None = ..., observed_metrics: collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics] @@ -1717,6 +1732,10 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"metrics", "ml_command_result", b"ml_command_result", + "pipeline_command_result", + b"pipeline_command_result", + "pipeline_event_result", + b"pipeline_event_result", "response_type", b"response_type", "result_complete", @@ -1758,6 +1777,10 @@ class ExecutePlanResponse(google.protobuf.message.Message): b"observed_metrics", "operation_id", b"operation_id", + "pipeline_command_result", + b"pipeline_command_result", + "pipeline_event_result", + b"pipeline_event_result", "response_id", b"response_id", "response_type", @@ -1798,6 +1821,8 @@ class ExecutePlanResponse(google.protobuf.message.Message): "execution_progress", "checkpoint_command_result", "ml_command_result", + "pipeline_event_result", + "pipeline_command_result", "extension", ] | None diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index d2f557aeaf02d..4339514a6835e 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -39,10 +39,11 @@ from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2 from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2 +from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto"\xae\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xca\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\xdc\x06\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\xd8\x06\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xfb\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12k\n\x1cwrite_stream_operation_start\x18\x06 \x01(\x0b\x32(.spark.connect.WriteStreamOperationStartH\x00R\x19writeStreamOperationStart\x12^\n\x17streaming_query_command\x18\x07 \x01(\x0b\x32$.spark.connect.StreamingQueryCommandH\x00R\x15streamingQueryCommand\x12X\n\x15get_resources_command\x18\x08 \x01(\x0b\x32".spark.connect.GetResourcesCommandH\x00R\x13getResourcesCommand\x12t\n\x1fstreaming_query_manager_command\x18\t \x01(\x0b\x32+.spark.connect.StreamingQueryManagerCommandH\x00R\x1cstreamingQueryManagerCommand\x12m\n\x17register_table_function\x18\n \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R\x15registerTableFunction\x12\x81\x01\n$streaming_query_listener_bus_command\x18\x0b \x01(\x0b\x32/.spark.connect.StreamingQueryListenerBusCommandH\x00R streamingQueryListenerBusCommand\x12\x64\n\x14register_data_source\x18\x0c \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R\x12registerDataSource\x12t\n\x1f\x63reate_resource_profile_command\x18\r \x01(\x0b\x32+.spark.connect.CreateResourceProfileCommandH\x00R\x1c\x63reateResourceProfileCommand\x12Q\n\x12\x63heckpoint_command\x18\x0e \x01(\x0b\x32 .spark.connect.CheckpointCommandH\x00R\x11\x63heckpointCommand\x12\x84\x01\n%remove_cached_remote_relation_command\x18\x0f \x01(\x0b\x32\x30.spark.connect.RemoveCachedRemoteRelationCommandH\x00R!removeCachedRemoteRelationCommand\x12_\n\x18merge_into_table_command\x18\x10 \x01(\x0b\x32$.spark.connect.MergeIntoTableCommandH\x00R\x15mergeIntoTableCommand\x12\x39\n\nml_command\x18\x11 \x01(\x0b\x32\x18.spark.connect.MlCommandH\x00R\tmlCommand\x12\x61\n\x18\x65xecute_external_command\x18\x12 \x01(\x0b\x32%.spark.connect.ExecuteExternalCommandH\x00R\x16\x65xecuteExternalCommand\x12K\n\x10pipeline_command\x18\x13 \x01(\x0b\x32\x1e.spark.connect.PipelineCommandH\x00R\x0fpipelineCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\xaa\x04\n\nSqlCommand\x12\x14\n\x03sql\x18\x01 \x01(\tB\x02\x18\x01R\x03sql\x12;\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12Z\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32-.spark.connect.SqlCommand.NamedArgumentsEntryB\x02\x18\x01R\x0enamedArguments\x12\x42\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionB\x02\x18\x01R\x0cposArguments\x12-\n\x05input\x18\x06 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xca\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x12-\n\x12\x63lustering_columns\x18\n \x03(\tR\x11\x63lusteringColumns\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\xdc\x06\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x12-\n\x12\x63lustering_columns\x18\t \x03(\tR\x11\x63lusteringColumns\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_provider"\xd8\x06\n\x19WriteStreamOperationStart\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06\x66ormat\x18\x02 \x01(\tR\x06\x66ormat\x12O\n\x07options\x18\x03 \x03(\x0b\x32\x35.spark.connect.WriteStreamOperationStart.OptionsEntryR\x07options\x12:\n\x19partitioning_column_names\x18\x04 \x03(\tR\x17partitioningColumnNames\x12:\n\x18processing_time_interval\x18\x05 \x01(\tH\x00R\x16processingTimeInterval\x12%\n\ravailable_now\x18\x06 \x01(\x08H\x00R\x0c\x61vailableNow\x12\x14\n\x04once\x18\x07 \x01(\x08H\x00R\x04once\x12\x46\n\x1e\x63ontinuous_checkpoint_interval\x18\x08 \x01(\tH\x00R\x1c\x63ontinuousCheckpointInterval\x12\x1f\n\x0boutput_mode\x18\t \x01(\tR\noutputMode\x12\x1d\n\nquery_name\x18\n \x01(\tR\tqueryName\x12\x14\n\x04path\x18\x0b \x01(\tH\x01R\x04path\x12\x1f\n\ntable_name\x18\x0c \x01(\tH\x01R\ttableName\x12N\n\x0e\x66oreach_writer\x18\r \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\rforeachWriter\x12L\n\rforeach_batch\x18\x0e \x01(\x0b\x32\'.spark.connect.StreamingForeachFunctionR\x0c\x66oreachBatch\x12\x36\n\x17\x63lustering_column_names\x18\x0f \x03(\tR\x15\x63lusteringColumnNames\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07triggerB\x12\n\x10sink_destination"\xb3\x01\n\x18StreamingForeachFunction\x12\x43\n\x0fpython_function\x18\x01 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x0epythonFunction\x12\x46\n\x0escala_function\x18\x02 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\rscalaFunctionB\n\n\x08\x66unction"\xd4\x01\n\x1fWriteStreamOperationStartResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12<\n\x18query_started_event_json\x18\x03 \x01(\tH\x00R\x15queryStartedEventJson\x88\x01\x01\x42\x1b\n\x19_query_started_event_json"A\n\x18StreamingQueryInstanceId\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x15\n\x06run_id\x18\x02 \x01(\tR\x05runId"\xf8\x04\n\x15StreamingQueryCommand\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12\x18\n\x06status\x18\x02 \x01(\x08H\x00R\x06status\x12%\n\rlast_progress\x18\x03 \x01(\x08H\x00R\x0clastProgress\x12)\n\x0frecent_progress\x18\x04 \x01(\x08H\x00R\x0erecentProgress\x12\x14\n\x04stop\x18\x05 \x01(\x08H\x00R\x04stop\x12\x34\n\x15process_all_available\x18\x06 \x01(\x08H\x00R\x13processAllAvailable\x12O\n\x07\x65xplain\x18\x07 \x01(\x0b\x32\x33.spark.connect.StreamingQueryCommand.ExplainCommandH\x00R\x07\x65xplain\x12\x1e\n\texception\x18\x08 \x01(\x08H\x00R\texception\x12k\n\x11\x61wait_termination\x18\t \x01(\x0b\x32<.spark.connect.StreamingQueryCommand.AwaitTerminationCommandH\x00R\x10\x61waitTermination\x1a,\n\x0e\x45xplainCommand\x12\x1a\n\x08\x65xtended\x18\x01 \x01(\x08R\x08\x65xtended\x1aL\n\x17\x41waitTerminationCommand\x12"\n\ntimeout_ms\x18\x02 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_msB\t\n\x07\x63ommand"\xf5\x08\n\x1bStreamingQueryCommandResult\x12\x42\n\x08query_id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x07queryId\x12Q\n\x06status\x18\x02 \x01(\x0b\x32\x37.spark.connect.StreamingQueryCommandResult.StatusResultH\x00R\x06status\x12j\n\x0frecent_progress\x18\x03 \x01(\x0b\x32?.spark.connect.StreamingQueryCommandResult.RecentProgressResultH\x00R\x0erecentProgress\x12T\n\x07\x65xplain\x18\x04 \x01(\x0b\x32\x38.spark.connect.StreamingQueryCommandResult.ExplainResultH\x00R\x07\x65xplain\x12Z\n\texception\x18\x05 \x01(\x0b\x32:.spark.connect.StreamingQueryCommandResult.ExceptionResultH\x00R\texception\x12p\n\x11\x61wait_termination\x18\x06 \x01(\x0b\x32\x41.spark.connect.StreamingQueryCommandResult.AwaitTerminationResultH\x00R\x10\x61waitTermination\x1a\xaa\x01\n\x0cStatusResult\x12%\n\x0estatus_message\x18\x01 \x01(\tR\rstatusMessage\x12*\n\x11is_data_available\x18\x02 \x01(\x08R\x0fisDataAvailable\x12*\n\x11is_trigger_active\x18\x03 \x01(\x08R\x0fisTriggerActive\x12\x1b\n\tis_active\x18\x04 \x01(\x08R\x08isActive\x1aH\n\x14RecentProgressResult\x12\x30\n\x14recent_progress_json\x18\x05 \x03(\tR\x12recentProgressJson\x1a\'\n\rExplainResult\x12\x16\n\x06result\x18\x01 \x01(\tR\x06result\x1a\xc5\x01\n\x0f\x45xceptionResult\x12\x30\n\x11\x65xception_message\x18\x01 \x01(\tH\x00R\x10\x65xceptionMessage\x88\x01\x01\x12$\n\x0b\x65rror_class\x18\x02 \x01(\tH\x01R\nerrorClass\x88\x01\x01\x12$\n\x0bstack_trace\x18\x03 \x01(\tH\x02R\nstackTrace\x88\x01\x01\x42\x14\n\x12_exception_messageB\x0e\n\x0c_error_classB\x0e\n\x0c_stack_trace\x1a\x38\n\x16\x41waitTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminatedB\r\n\x0bresult_type"\xbd\x06\n\x1cStreamingQueryManagerCommand\x12\x18\n\x06\x61\x63tive\x18\x01 \x01(\x08H\x00R\x06\x61\x63tive\x12\x1d\n\tget_query\x18\x02 \x01(\tH\x00R\x08getQuery\x12|\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32\x46.spark.connect.StreamingQueryManagerCommand.AwaitAnyTerminationCommandH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12n\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0b\x61\x64\x64Listener\x12t\n\x0fremove_listener\x18\x06 \x01(\x0b\x32I.spark.connect.StreamingQueryManagerCommand.StreamingQueryListenerCommandH\x00R\x0eremoveListener\x12\'\n\x0elist_listeners\x18\x07 \x01(\x08H\x00R\rlistListeners\x1aO\n\x1a\x41waitAnyTerminationCommand\x12"\n\ntimeout_ms\x18\x01 \x01(\x03H\x00R\ttimeoutMs\x88\x01\x01\x42\r\n\x0b_timeout_ms\x1a\xcd\x01\n\x1dStreamingQueryListenerCommand\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x12U\n\x17python_listener_payload\x18\x02 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\x15pythonListenerPayload\x88\x01\x01\x12\x0e\n\x02id\x18\x03 \x01(\tR\x02idB\x1a\n\x18_python_listener_payloadB\t\n\x07\x63ommand"\xb4\x08\n"StreamingQueryManagerCommandResult\x12X\n\x06\x61\x63tive\x18\x01 \x01(\x0b\x32>.spark.connect.StreamingQueryManagerCommandResult.ActiveResultH\x00R\x06\x61\x63tive\x12`\n\x05query\x18\x02 \x01(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceH\x00R\x05query\x12\x81\x01\n\x15\x61wait_any_termination\x18\x03 \x01(\x0b\x32K.spark.connect.StreamingQueryManagerCommandResult.AwaitAnyTerminationResultH\x00R\x13\x61waitAnyTermination\x12+\n\x10reset_terminated\x18\x04 \x01(\x08H\x00R\x0fresetTerminated\x12#\n\x0c\x61\x64\x64_listener\x18\x05 \x01(\x08H\x00R\x0b\x61\x64\x64Listener\x12)\n\x0fremove_listener\x18\x06 \x01(\x08H\x00R\x0eremoveListener\x12{\n\x0elist_listeners\x18\x07 \x01(\x0b\x32R.spark.connect.StreamingQueryManagerCommandResult.ListStreamingQueryListenerResultH\x00R\rlistListeners\x1a\x7f\n\x0c\x41\x63tiveResult\x12o\n\x0e\x61\x63tive_queries\x18\x01 \x03(\x0b\x32H.spark.connect.StreamingQueryManagerCommandResult.StreamingQueryInstanceR\ractiveQueries\x1as\n\x16StreamingQueryInstance\x12\x37\n\x02id\x18\x01 \x01(\x0b\x32\'.spark.connect.StreamingQueryInstanceIdR\x02id\x12\x17\n\x04name\x18\x02 \x01(\tH\x00R\x04name\x88\x01\x01\x42\x07\n\x05_name\x1a;\n\x19\x41waitAnyTerminationResult\x12\x1e\n\nterminated\x18\x01 \x01(\x08R\nterminated\x1aK\n\x1eStreamingQueryListenerInstance\x12)\n\x10listener_payload\x18\x01 \x01(\x0cR\x0flistenerPayload\x1a\x45\n ListStreamingQueryListenerResult\x12!\n\x0clistener_ids\x18\x01 \x03(\tR\x0blistenerIdsB\r\n\x0bresult_type"\xad\x01\n StreamingQueryListenerBusCommand\x12;\n\x19\x61\x64\x64_listener_bus_listener\x18\x01 \x01(\x08H\x00R\x16\x61\x64\x64ListenerBusListener\x12\x41\n\x1cremove_listener_bus_listener\x18\x02 \x01(\x08H\x00R\x19removeListenerBusListenerB\t\n\x07\x63ommand"\x83\x01\n\x1bStreamingQueryListenerEvent\x12\x1d\n\nevent_json\x18\x01 \x01(\tR\teventJson\x12\x45\n\nevent_type\x18\x02 \x01(\x0e\x32&.spark.connect.StreamingQueryEventTypeR\teventType"\xcc\x01\n"StreamingQueryListenerEventsResult\x12\x42\n\x06\x65vents\x18\x01 \x03(\x0b\x32*.spark.connect.StreamingQueryListenerEventR\x06\x65vents\x12\x42\n\x1blistener_bus_listener_added\x18\x02 \x01(\x08H\x00R\x18listenerBusListenerAdded\x88\x01\x01\x42\x1e\n\x1c_listener_bus_listener_added"\x15\n\x13GetResourcesCommand"\xd4\x01\n\x19GetResourcesCommandResult\x12U\n\tresources\x18\x01 \x03(\x0b\x32\x37.spark.connect.GetResourcesCommandResult.ResourcesEntryR\tresources\x1a`\n\x0eResourcesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x38\n\x05value\x18\x02 \x01(\x0b\x32".spark.connect.ResourceInformationR\x05value:\x02\x38\x01"X\n\x1c\x43reateResourceProfileCommand\x12\x38\n\x07profile\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ResourceProfileR\x07profile"C\n"CreateResourceProfileCommandResult\x12\x1d\n\nprofile_id\x18\x01 \x01(\x05R\tprofileId"d\n!RemoveCachedRemoteRelationCommand\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation"\xcd\x01\n\x11\x43heckpointCommand\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x14\n\x05local\x18\x02 \x01(\x08R\x05local\x12\x14\n\x05\x65\x61ger\x18\x03 \x01(\x08R\x05\x65\x61ger\x12\x45\n\rstorage_level\x18\x04 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level"\xe8\x03\n\x15MergeIntoTableCommand\x12*\n\x11target_table_name\x18\x01 \x01(\tR\x0ftargetTableName\x12\x43\n\x11source_table_plan\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x0fsourceTablePlan\x12\x42\n\x0fmerge_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0emergeCondition\x12>\n\rmatch_actions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cmatchActions\x12I\n\x13not_matched_actions\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11notMatchedActions\x12[\n\x1dnot_matched_by_source_actions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x19notMatchedBySourceActions\x12\x32\n\x15with_schema_evolution\x18\x07 \x01(\x08R\x13withSchemaEvolution"\xd4\x01\n\x16\x45xecuteExternalCommand\x12\x16\n\x06runner\x18\x01 \x01(\tR\x06runner\x12\x18\n\x07\x63ommand\x18\x02 \x01(\tR\x07\x63ommand\x12L\n\x07options\x18\x03 \x03(\x0b\x32\x32.spark.connect.ExecuteExternalCommand.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01*\x85\x01\n\x17StreamingQueryEventType\x12\x1e\n\x1aQUERY_PROGRESS_UNSPECIFIED\x10\x00\x12\x18\n\x14QUERY_PROGRESS_EVENT\x10\x01\x12\x1a\n\x16QUERY_TERMINATED_EVENT\x10\x02\x12\x14\n\x10QUERY_IDLE_EVENT\x10\x03\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -81,118 +82,118 @@ _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_options = b"8\001" _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._loaded_options = None _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_options = b"8\001" - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 11649 - _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 11782 - _globals["_COMMAND"]._serialized_start = 191 - _globals["_COMMAND"]._serialized_end = 2029 - _globals["_SQLCOMMAND"]._serialized_start = 2032 - _globals["_SQLCOMMAND"]._serialized_end = 2586 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2402 - _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 2492 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 2494 - _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 2586 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 2589 - _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 2739 - _globals["_WRITEOPERATION"]._serialized_start = 2742 - _globals["_WRITEOPERATION"]._serialized_end = 3840 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3264 - _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3322 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3325 - _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 3583 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 3459 - _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 3583 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 3585 - _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 3676 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 3679 - _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 3816 - _globals["_WRITEOPERATIONV2"]._serialized_start = 3843 - _globals["_WRITEOPERATIONV2"]._serialized_end = 4703 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3264 - _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3322 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 4462 - _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 4528 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 4531 - _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 4690 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 4706 - _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 5562 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3264 - _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3322 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 5565 - _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 5744 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 5747 - _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 5959 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 5961 - _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6026 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6029 - _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 6661 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 6528 - _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 6572 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 6574 - _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 6650 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 6664 - _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 7805 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7247 - _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 7417 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 7419 - _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 7491 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 7493 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 7532 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 7535 - _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 7732 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 7734 - _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 7790 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 7808 - _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 8637 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 8339 - _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 8418 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 8421 - _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 8626 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 8640 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 9716 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9248 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9375 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9377 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9492 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_start = 11757 + _globals["_STREAMINGQUERYEVENTTYPE"]._serialized_end = 11890 + _globals["_COMMAND"]._serialized_start = 222 + _globals["_COMMAND"]._serialized_end = 2137 + _globals["_SQLCOMMAND"]._serialized_start = 2140 + _globals["_SQLCOMMAND"]._serialized_end = 2694 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_start = 2510 + _globals["_SQLCOMMAND_ARGSENTRY"]._serialized_end = 2600 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_start = 2602 + _globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._serialized_end = 2694 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_start = 2697 + _globals["_CREATEDATAFRAMEVIEWCOMMAND"]._serialized_end = 2847 + _globals["_WRITEOPERATION"]._serialized_start = 2850 + _globals["_WRITEOPERATION"]._serialized_end = 3948 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_start = 3372 + _globals["_WRITEOPERATION_OPTIONSENTRY"]._serialized_end = 3430 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_start = 3433 + _globals["_WRITEOPERATION_SAVETABLE"]._serialized_end = 3691 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_start = 3567 + _globals["_WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD"]._serialized_end = 3691 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_start = 3693 + _globals["_WRITEOPERATION_BUCKETBY"]._serialized_end = 3784 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_start = 3787 + _globals["_WRITEOPERATION_SAVEMODE"]._serialized_end = 3924 + _globals["_WRITEOPERATIONV2"]._serialized_start = 3951 + _globals["_WRITEOPERATIONV2"]._serialized_end = 4811 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_start = 3372 + _globals["_WRITEOPERATIONV2_OPTIONSENTRY"]._serialized_end = 3430 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_start = 4570 + _globals["_WRITEOPERATIONV2_TABLEPROPERTIESENTRY"]._serialized_end = 4636 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_start = 4639 + _globals["_WRITEOPERATIONV2_MODE"]._serialized_end = 4798 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_start = 4814 + _globals["_WRITESTREAMOPERATIONSTART"]._serialized_end = 5670 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_start = 3372 + _globals["_WRITESTREAMOPERATIONSTART_OPTIONSENTRY"]._serialized_end = 3430 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_start = 5673 + _globals["_STREAMINGFOREACHFUNCTION"]._serialized_end = 5852 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_start = 5855 + _globals["_WRITESTREAMOPERATIONSTARTRESULT"]._serialized_end = 6067 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_start = 6069 + _globals["_STREAMINGQUERYINSTANCEID"]._serialized_end = 6134 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_start = 6137 + _globals["_STREAMINGQUERYCOMMAND"]._serialized_end = 6769 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_start = 6636 + _globals["_STREAMINGQUERYCOMMAND_EXPLAINCOMMAND"]._serialized_end = 6680 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_start = 6682 + _globals["_STREAMINGQUERYCOMMAND_AWAITTERMINATIONCOMMAND"]._serialized_end = 6758 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_start = 6772 + _globals["_STREAMINGQUERYCOMMANDRESULT"]._serialized_end = 7913 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_start = 7355 + _globals["_STREAMINGQUERYCOMMANDRESULT_STATUSRESULT"]._serialized_end = 7525 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_start = 7527 + _globals["_STREAMINGQUERYCOMMANDRESULT_RECENTPROGRESSRESULT"]._serialized_end = 7599 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_start = 7601 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXPLAINRESULT"]._serialized_end = 7640 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_start = 7643 + _globals["_STREAMINGQUERYCOMMANDRESULT_EXCEPTIONRESULT"]._serialized_end = 7840 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_start = 7842 + _globals["_STREAMINGQUERYCOMMANDRESULT_AWAITTERMINATIONRESULT"]._serialized_end = 7898 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_start = 7916 + _globals["_STREAMINGQUERYMANAGERCOMMAND"]._serialized_end = 8745 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_start = 8447 + _globals["_STREAMINGQUERYMANAGERCOMMAND_AWAITANYTERMINATIONCOMMAND"]._serialized_end = 8526 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_start = 8529 + _globals["_STREAMINGQUERYMANAGERCOMMAND_STREAMINGQUERYLISTENERCOMMAND"]._serialized_end = 8734 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_start = 8748 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT"]._serialized_end = 9824 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_start = 9356 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9483 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9485 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9600 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT" - ]._serialized_start = 9494 - _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9553 + ]._serialized_start = 9602 + _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9661 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_start = 9555 + ]._serialized_start = 9663 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE" - ]._serialized_end = 9630 + ]._serialized_end = 9738 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_start = 9632 + ]._serialized_start = 9740 _globals[ "_STREAMINGQUERYMANAGERCOMMANDRESULT_LISTSTREAMINGQUERYLISTENERRESULT" - ]._serialized_end = 9701 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 9719 - _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 9892 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 9895 - _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10026 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10029 - _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10233 - _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10235 - _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10256 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10259 - _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 10471 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 10375 - _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 10471 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 10473 - _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 10561 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 10563 - _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 10630 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 10632 - _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 10732 - _globals["_CHECKPOINTCOMMAND"]._serialized_start = 10735 - _globals["_CHECKPOINTCOMMAND"]._serialized_end = 10940 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 10943 - _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 11431 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 11434 - _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 11646 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3264 - _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3322 + ]._serialized_end = 9809 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_start = 9827 + _globals["_STREAMINGQUERYLISTENERBUSCOMMAND"]._serialized_end = 10000 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_start = 10003 + _globals["_STREAMINGQUERYLISTENEREVENT"]._serialized_end = 10134 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_start = 10137 + _globals["_STREAMINGQUERYLISTENEREVENTSRESULT"]._serialized_end = 10341 + _globals["_GETRESOURCESCOMMAND"]._serialized_start = 10343 + _globals["_GETRESOURCESCOMMAND"]._serialized_end = 10364 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_start = 10367 + _globals["_GETRESOURCESCOMMANDRESULT"]._serialized_end = 10579 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_start = 10483 + _globals["_GETRESOURCESCOMMANDRESULT_RESOURCESENTRY"]._serialized_end = 10579 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_start = 10581 + _globals["_CREATERESOURCEPROFILECOMMAND"]._serialized_end = 10669 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_start = 10671 + _globals["_CREATERESOURCEPROFILECOMMANDRESULT"]._serialized_end = 10738 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_start = 10740 + _globals["_REMOVECACHEDREMOTERELATIONCOMMAND"]._serialized_end = 10840 + _globals["_CHECKPOINTCOMMAND"]._serialized_start = 10843 + _globals["_CHECKPOINTCOMMAND"]._serialized_end = 11048 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_start = 11051 + _globals["_MERGEINTOTABLECOMMAND"]._serialized_end = 11539 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_start = 11542 + _globals["_EXECUTEEXTERNALCOMMAND"]._serialized_end = 11754 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_start = 3372 + _globals["_EXECUTEEXTERNALCOMMAND_OPTIONSENTRY"]._serialized_end = 3430 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index 9ca0607abd942..66e0ffdb52731 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -43,6 +43,7 @@ import google.protobuf.message import pyspark.sql.connect.proto.common_pb2 import pyspark.sql.connect.proto.expressions_pb2 import pyspark.sql.connect.proto.ml_pb2 +import pyspark.sql.connect.proto.pipelines_pb2 import pyspark.sql.connect.proto.relations_pb2 import sys import typing @@ -107,6 +108,7 @@ class Command(google.protobuf.message.Message): MERGE_INTO_TABLE_COMMAND_FIELD_NUMBER: builtins.int ML_COMMAND_FIELD_NUMBER: builtins.int EXECUTE_EXTERNAL_COMMAND_FIELD_NUMBER: builtins.int + PIPELINE_COMMAND_FIELD_NUMBER: builtins.int EXTENSION_FIELD_NUMBER: builtins.int @property def register_function( @@ -153,6 +155,8 @@ class Command(google.protobuf.message.Message): @property def execute_external_command(self) -> global___ExecuteExternalCommand: ... @property + def pipeline_command(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand: ... + @property def extension(self) -> google.protobuf.any_pb2.Any: """This field is used to mark extensions to the protocol. When plugins generate arbitrary Commands they can add them here. During the planning the correct resolution is done. @@ -183,6 +187,7 @@ class Command(google.protobuf.message.Message): merge_into_table_command: global___MergeIntoTableCommand | None = ..., ml_command: pyspark.sql.connect.proto.ml_pb2.MlCommand | None = ..., execute_external_command: global___ExecuteExternalCommand | None = ..., + pipeline_command: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommand | None = ..., extension: google.protobuf.any_pb2.Any | None = ..., ) -> None: ... def HasField( @@ -206,6 +211,8 @@ class Command(google.protobuf.message.Message): b"merge_into_table_command", "ml_command", b"ml_command", + "pipeline_command", + b"pipeline_command", "register_data_source", b"register_data_source", "register_function", @@ -251,6 +258,8 @@ class Command(google.protobuf.message.Message): b"merge_into_table_command", "ml_command", b"ml_command", + "pipeline_command", + b"pipeline_command", "register_data_source", b"register_data_source", "register_function", @@ -297,6 +306,7 @@ class Command(google.protobuf.message.Message): "merge_into_table_command", "ml_command", "execute_external_command", + "pipeline_command", "extension", ] | None diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.py b/python/pyspark/sql/connect/proto/pipelines_pb2.py new file mode 100644 index 0000000000000..04ff64796961e --- /dev/null +++ b/python/pyspark/sql/connect/proto/pipelines_pb2.py @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: spark/connect/pipelines.proto +# Protobuf Python Version: 5.28.3 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 5, 28, 3, "", "spark/connect/pipelines.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2 +from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x1dspark/connect/pipelines.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\x8c\x11\n\x0fPipelineCommand\x12h\n\x15\x63reate_dataflow_graph\x18\x01 \x01(\x0b\x32\x32.spark.connect.PipelineCommand.CreateDataflowGraphH\x00R\x13\x63reateDataflowGraph\x12U\n\x0e\x64\x65\x66ine_dataset\x18\x02 \x01(\x0b\x32,.spark.connect.PipelineCommand.DefineDatasetH\x00R\rdefineDataset\x12L\n\x0b\x64\x65\x66ine_flow\x18\x03 \x01(\x0b\x32).spark.connect.PipelineCommand.DefineFlowH\x00R\ndefineFlow\x12\x62\n\x13\x64rop_dataflow_graph\x18\x04 \x01(\x0b\x32\x30.spark.connect.PipelineCommand.DropDataflowGraphH\x00R\x11\x64ropDataflowGraph\x12\x46\n\tstart_run\x18\x05 \x01(\x0b\x32\'.spark.connect.PipelineCommand.StartRunH\x00R\x08startRun\x12\x62\n\x19\x64\x65\x66ine_sql_graph_elements\x18\x06 \x01(\x0b\x32%.spark.connect.DefineSqlGraphElementsH\x00R\x16\x64\x65\x66ineSqlGraphElements\x1a\x87\x03\n\x13\x43reateDataflowGraph\x12,\n\x0f\x64\x65\x66\x61ult_catalog\x18\x01 \x01(\tH\x00R\x0e\x64\x65\x66\x61ultCatalog\x88\x01\x01\x12.\n\x10\x64\x65\x66\x61ult_database\x18\x02 \x01(\tH\x01R\x0f\x64\x65\x66\x61ultDatabase\x88\x01\x01\x12Z\n\x08sql_conf\x18\x05 \x03(\x0b\x32?.spark.connect.PipelineCommand.CreateDataflowGraph.SqlConfEntryR\x07sqlConf\x1a:\n\x0cSqlConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1aQ\n\x08Response\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x42\x14\n\x12_dataflow_graph_idB\x12\n\x10_default_catalogB\x13\n\x11_default_database\x1aZ\n\x11\x44ropDataflowGraph\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x42\x14\n\x12_dataflow_graph_id\x1a\xd1\x04\n\rDefineDataset\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x12&\n\x0c\x64\x61taset_name\x18\x02 \x01(\tH\x01R\x0b\x64\x61tasetName\x88\x01\x01\x12\x42\n\x0c\x64\x61taset_type\x18\x03 \x01(\x0e\x32\x1a.spark.connect.DatasetTypeH\x02R\x0b\x64\x61tasetType\x88\x01\x01\x12\x1d\n\x07\x63omment\x18\x04 \x01(\tH\x03R\x07\x63omment\x88\x01\x01\x12l\n\x10table_properties\x18\x05 \x03(\x0b\x32\x41.spark.connect.PipelineCommand.DefineDataset.TablePropertiesEntryR\x0ftableProperties\x12%\n\x0epartition_cols\x18\x06 \x03(\tR\rpartitionCols\x12\x34\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x04R\x06schema\x88\x01\x01\x12\x1b\n\x06\x66ormat\x18\x08 \x01(\tH\x05R\x06\x66ormat\x88\x01\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x14\n\x12_dataflow_graph_idB\x0f\n\r_dataset_nameB\x0f\n\r_dataset_typeB\n\n\x08_commentB\t\n\x07_schemaB\t\n\x07_format\x1a\xbc\x03\n\nDefineFlow\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x12 \n\tflow_name\x18\x02 \x01(\tH\x01R\x08\x66lowName\x88\x01\x01\x12\x33\n\x13target_dataset_name\x18\x03 \x01(\tH\x02R\x11targetDatasetName\x88\x01\x01\x12\x30\n\x04plan\x18\x04 \x01(\x0b\x32\x17.spark.connect.RelationH\x03R\x04plan\x88\x01\x01\x12Q\n\x08sql_conf\x18\x05 \x03(\x0b\x32\x36.spark.connect.PipelineCommand.DefineFlow.SqlConfEntryR\x07sqlConf\x12\x17\n\x04once\x18\x06 \x01(\x08H\x04R\x04once\x88\x01\x01\x1a:\n\x0cSqlConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x14\n\x12_dataflow_graph_idB\x0c\n\n_flow_nameB\x16\n\x14_target_dataset_nameB\x07\n\x05_planB\x07\n\x05_once\x1aQ\n\x08StartRun\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x42\x14\n\x12_dataflow_graph_idB\x0e\n\x0c\x63ommand_type"\xc7\x01\n\x16\x44\x65\x66ineSqlGraphElements\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x12\'\n\rsql_file_path\x18\x02 \x01(\tH\x01R\x0bsqlFilePath\x88\x01\x01\x12\x1e\n\x08sql_text\x18\x03 \x01(\tH\x02R\x07sqlText\x88\x01\x01\x42\x14\n\x12_dataflow_graph_idB\x10\n\x0e_sql_file_pathB\x0b\n\t_sql_text"\x8e\x02\n\x15PipelineCommandResult\x12\x81\x01\n\x1c\x63reate_dataflow_graph_result\x18\x01 \x01(\x0b\x32>.spark.connect.PipelineCommandResult.CreateDataflowGraphResultH\x00R\x19\x63reateDataflowGraphResult\x1a\x62\n\x19\x43reateDataflowGraphResult\x12/\n\x11\x64\x61taflow_graph_id\x18\x01 \x01(\tH\x00R\x0f\x64\x61taflowGraphId\x88\x01\x01\x42\x14\n\x12_dataflow_graph_idB\r\n\x0bresult_type"I\n\x13PipelineEventResult\x12\x32\n\x05\x65vent\x18\x01 \x01(\x0b\x32\x1c.spark.connect.PipelineEventR\x05\x65vent"k\n\rPipelineEvent\x12!\n\ttimestamp\x18\x01 \x01(\tH\x00R\ttimestamp\x88\x01\x01\x12\x1d\n\x07message\x18\x02 \x01(\tH\x01R\x07message\x88\x01\x01\x42\x0c\n\n_timestampB\n\n\x08_message*a\n\x0b\x44\x61tasetType\x12\x1c\n\x18\x44\x41TASET_TYPE_UNSPECIFIED\x10\x00\x12\x15\n\x11MATERIALIZED_VIEW\x10\x01\x12\t\n\x05TABLE\x10\x02\x12\x12\n\x0eTEMPORARY_VIEW\x10\x03\x42"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, "pyspark.sql.connect.proto.pipelines_pb2", _globals +) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001" + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._loaded_options = None + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._serialized_options = b"8\001" + _globals["_PIPELINECOMMAND_DEFINEDATASET_TABLEPROPERTIESENTRY"]._loaded_options = None + _globals["_PIPELINECOMMAND_DEFINEDATASET_TABLEPROPERTIESENTRY"]._serialized_options = b"8\001" + _globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._loaded_options = None + _globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._serialized_options = b"8\001" + _globals["_DATASETTYPE"]._serialized_start = 2956 + _globals["_DATASETTYPE"]._serialized_end = 3053 + _globals["_PIPELINECOMMAND"]._serialized_start = 107 + _globals["_PIPELINECOMMAND"]._serialized_end = 2295 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH"]._serialized_start = 670 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH"]._serialized_end = 1061 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._serialized_start = 879 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._serialized_end = 937 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_RESPONSE"]._serialized_start = 939 + _globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_RESPONSE"]._serialized_end = 1020 + _globals["_PIPELINECOMMAND_DROPDATAFLOWGRAPH"]._serialized_start = 1063 + _globals["_PIPELINECOMMAND_DROPDATAFLOWGRAPH"]._serialized_end = 1153 + _globals["_PIPELINECOMMAND_DEFINEDATASET"]._serialized_start = 1156 + _globals["_PIPELINECOMMAND_DEFINEDATASET"]._serialized_end = 1749 + _globals["_PIPELINECOMMAND_DEFINEDATASET_TABLEPROPERTIESENTRY"]._serialized_start = 1593 + _globals["_PIPELINECOMMAND_DEFINEDATASET_TABLEPROPERTIESENTRY"]._serialized_end = 1659 + _globals["_PIPELINECOMMAND_DEFINEFLOW"]._serialized_start = 1752 + _globals["_PIPELINECOMMAND_DEFINEFLOW"]._serialized_end = 2196 + _globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._serialized_start = 879 + _globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._serialized_end = 937 + _globals["_PIPELINECOMMAND_STARTRUN"]._serialized_start = 2198 + _globals["_PIPELINECOMMAND_STARTRUN"]._serialized_end = 2279 + _globals["_DEFINESQLGRAPHELEMENTS"]._serialized_start = 2298 + _globals["_DEFINESQLGRAPHELEMENTS"]._serialized_end = 2497 + _globals["_PIPELINECOMMANDRESULT"]._serialized_start = 2500 + _globals["_PIPELINECOMMANDRESULT"]._serialized_end = 2770 + _globals["_PIPELINECOMMANDRESULT_CREATEDATAFLOWGRAPHRESULT"]._serialized_start = 2657 + _globals["_PIPELINECOMMANDRESULT_CREATEDATAFLOWGRAPHRESULT"]._serialized_end = 2755 + _globals["_PIPELINEEVENTRESULT"]._serialized_start = 2772 + _globals["_PIPELINEEVENTRESULT"]._serialized_end = 2845 + _globals["_PIPELINEEVENT"]._serialized_start = 2847 + _globals["_PIPELINEEVENT"]._serialized_end = 2954 +# @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.pyi b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi new file mode 100644 index 0000000000000..6402a6593e707 --- /dev/null +++ b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi @@ -0,0 +1,860 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import pyspark.sql.connect.proto.relations_pb2 +import pyspark.sql.connect.proto.types_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _DatasetType: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _DatasetTypeEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_DatasetType.ValueType], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + DATASET_TYPE_UNSPECIFIED: _DatasetType.ValueType # 0 + """Safe default value. Should not be used.""" + MATERIALIZED_VIEW: _DatasetType.ValueType # 1 + """A materialized view dataset which is published to the catalog""" + TABLE: _DatasetType.ValueType # 2 + """A table which is published to the catalog""" + TEMPORARY_VIEW: _DatasetType.ValueType # 3 + """A view which is not published to the catalog""" + +class DatasetType(_DatasetType, metaclass=_DatasetTypeEnumTypeWrapper): + """The type of dataset.""" + +DATASET_TYPE_UNSPECIFIED: DatasetType.ValueType # 0 +"""Safe default value. Should not be used.""" +MATERIALIZED_VIEW: DatasetType.ValueType # 1 +"""A materialized view dataset which is published to the catalog""" +TABLE: DatasetType.ValueType # 2 +"""A table which is published to the catalog""" +TEMPORARY_VIEW: DatasetType.ValueType # 3 +"""A view which is not published to the catalog""" +global___DatasetType = DatasetType + +class PipelineCommand(google.protobuf.message.Message): + """Dispatch object for pipelines commands. See each individual command for documentation.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class CreateDataflowGraph(google.protobuf.message.Message): + """Request to create a new dataflow graph.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class SqlConfEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] + ) -> None: ... + + class Response(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The ID of the created graph.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> None: ... + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + + DEFAULT_CATALOG_FIELD_NUMBER: builtins.int + DEFAULT_DATABASE_FIELD_NUMBER: builtins.int + SQL_CONF_FIELD_NUMBER: builtins.int + default_catalog: builtins.str + """The default catalog.""" + default_database: builtins.str + """The default database.""" + @property + def sql_conf( + self, + ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """SQL configurations for all flows in this graph.""" + def __init__( + self, + *, + default_catalog: builtins.str | None = ..., + default_database: builtins.str | None = ..., + sql_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_default_catalog", + b"_default_catalog", + "_default_database", + b"_default_database", + "default_catalog", + b"default_catalog", + "default_database", + b"default_database", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_default_catalog", + b"_default_catalog", + "_default_database", + b"_default_database", + "default_catalog", + b"default_catalog", + "default_database", + b"default_database", + "sql_conf", + b"sql_conf", + ], + ) -> None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_default_catalog", b"_default_catalog"] + ) -> typing_extensions.Literal["default_catalog"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_default_database", b"_default_database"] + ) -> typing_extensions.Literal["default_database"] | None: ... + + class DropDataflowGraph(google.protobuf.message.Message): + """Drops the graph and stops any running attached flows.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The graph to drop.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> None: ... + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + + class DefineDataset(google.protobuf.message.Message): + """Request to define a dataset: a table, a materialized view, or a temporary view.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class TablePropertiesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] + ) -> None: ... + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + DATASET_NAME_FIELD_NUMBER: builtins.int + DATASET_TYPE_FIELD_NUMBER: builtins.int + COMMENT_FIELD_NUMBER: builtins.int + TABLE_PROPERTIES_FIELD_NUMBER: builtins.int + PARTITION_COLS_FIELD_NUMBER: builtins.int + SCHEMA_FIELD_NUMBER: builtins.int + FORMAT_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The graph to attach this dataset to.""" + dataset_name: builtins.str + """Name of the dataset. Can be partially or fully qualified.""" + dataset_type: global___DatasetType.ValueType + """The type of the dataset.""" + comment: builtins.str + """Optional comment for the dataset.""" + @property + def table_properties( + self, + ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """Optional table properties. Only applies to dataset_type == TABLE and dataset_type == MATERIALIZED_VIEW.""" + @property + def partition_cols( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Optional partition columns for the dataset. Only applies to dataset_type == TABLE and + dataset_type == MATERIALIZED_VIEW. + """ + @property + def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType: + """Schema for the dataset. If unset, this will be inferred from incoming flows.""" + format: builtins.str + """The output table format of the dataset. Only applies to dataset_type == TABLE and + dataset_type == MATERIALIZED_VIEW. + """ + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + dataset_name: builtins.str | None = ..., + dataset_type: global___DatasetType.ValueType | None = ..., + comment: builtins.str | None = ..., + table_properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + partition_cols: collections.abc.Iterable[builtins.str] | None = ..., + schema: pyspark.sql.connect.proto.types_pb2.DataType | None = ..., + format: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_comment", + b"_comment", + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_dataset_name", + b"_dataset_name", + "_dataset_type", + b"_dataset_type", + "_format", + b"_format", + "_schema", + b"_schema", + "comment", + b"comment", + "dataflow_graph_id", + b"dataflow_graph_id", + "dataset_name", + b"dataset_name", + "dataset_type", + b"dataset_type", + "format", + b"format", + "schema", + b"schema", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_comment", + b"_comment", + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_dataset_name", + b"_dataset_name", + "_dataset_type", + b"_dataset_type", + "_format", + b"_format", + "_schema", + b"_schema", + "comment", + b"comment", + "dataflow_graph_id", + b"dataflow_graph_id", + "dataset_name", + b"dataset_name", + "dataset_type", + b"dataset_type", + "format", + b"format", + "partition_cols", + b"partition_cols", + "schema", + b"schema", + "table_properties", + b"table_properties", + ], + ) -> None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_comment", b"_comment"] + ) -> typing_extensions.Literal["comment"] | None: ... + @typing.overload + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_dataset_name", b"_dataset_name"] + ) -> typing_extensions.Literal["dataset_name"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_dataset_type", b"_dataset_type"] + ) -> typing_extensions.Literal["dataset_type"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_format", b"_format"] + ) -> typing_extensions.Literal["format"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_schema", b"_schema"] + ) -> typing_extensions.Literal["schema"] | None: ... + + class DefineFlow(google.protobuf.message.Message): + """Request to define a flow targeting a dataset.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class SqlConfEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] + ) -> None: ... + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + FLOW_NAME_FIELD_NUMBER: builtins.int + TARGET_DATASET_NAME_FIELD_NUMBER: builtins.int + PLAN_FIELD_NUMBER: builtins.int + SQL_CONF_FIELD_NUMBER: builtins.int + ONCE_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The graph to attach this flow to.""" + flow_name: builtins.str + """Name of the flow. For standalone flows, this must be a single-part name.""" + target_dataset_name: builtins.str + """Name of the dataset this flow writes to. Can be partially or fully qualified.""" + @property + def plan(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: + """An unresolved relation that defines the dataset's flow.""" + @property + def sql_conf( + self, + ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """SQL configurations set when running this flow.""" + once: builtins.bool + """If true, this flow will only be run once per full refresh.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + flow_name: builtins.str | None = ..., + target_dataset_name: builtins.str | None = ..., + plan: pyspark.sql.connect.proto.relations_pb2.Relation | None = ..., + sql_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + once: builtins.bool | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_flow_name", + b"_flow_name", + "_once", + b"_once", + "_plan", + b"_plan", + "_target_dataset_name", + b"_target_dataset_name", + "dataflow_graph_id", + b"dataflow_graph_id", + "flow_name", + b"flow_name", + "once", + b"once", + "plan", + b"plan", + "target_dataset_name", + b"target_dataset_name", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_flow_name", + b"_flow_name", + "_once", + b"_once", + "_plan", + b"_plan", + "_target_dataset_name", + b"_target_dataset_name", + "dataflow_graph_id", + b"dataflow_graph_id", + "flow_name", + b"flow_name", + "once", + b"once", + "plan", + b"plan", + "sql_conf", + b"sql_conf", + "target_dataset_name", + b"target_dataset_name", + ], + ) -> None: ... + @typing.overload + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_flow_name", b"_flow_name"] + ) -> typing_extensions.Literal["flow_name"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_once", b"_once"] + ) -> typing_extensions.Literal["once"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_plan", b"_plan"] + ) -> typing_extensions.Literal["plan"] | None: ... + @typing.overload + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_target_dataset_name", b"_target_dataset_name"], + ) -> typing_extensions.Literal["target_dataset_name"] | None: ... + + class StartRun(google.protobuf.message.Message): + """Resolves all datasets and flows and start a pipeline update. Should be called after all + graph elements are registered. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The graph to start.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> None: ... + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + + CREATE_DATAFLOW_GRAPH_FIELD_NUMBER: builtins.int + DEFINE_DATASET_FIELD_NUMBER: builtins.int + DEFINE_FLOW_FIELD_NUMBER: builtins.int + DROP_DATAFLOW_GRAPH_FIELD_NUMBER: builtins.int + START_RUN_FIELD_NUMBER: builtins.int + DEFINE_SQL_GRAPH_ELEMENTS_FIELD_NUMBER: builtins.int + @property + def create_dataflow_graph(self) -> global___PipelineCommand.CreateDataflowGraph: ... + @property + def define_dataset(self) -> global___PipelineCommand.DefineDataset: ... + @property + def define_flow(self) -> global___PipelineCommand.DefineFlow: ... + @property + def drop_dataflow_graph(self) -> global___PipelineCommand.DropDataflowGraph: ... + @property + def start_run(self) -> global___PipelineCommand.StartRun: ... + @property + def define_sql_graph_elements(self) -> global___DefineSqlGraphElements: ... + def __init__( + self, + *, + create_dataflow_graph: global___PipelineCommand.CreateDataflowGraph | None = ..., + define_dataset: global___PipelineCommand.DefineDataset | None = ..., + define_flow: global___PipelineCommand.DefineFlow | None = ..., + drop_dataflow_graph: global___PipelineCommand.DropDataflowGraph | None = ..., + start_run: global___PipelineCommand.StartRun | None = ..., + define_sql_graph_elements: global___DefineSqlGraphElements | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "command_type", + b"command_type", + "create_dataflow_graph", + b"create_dataflow_graph", + "define_dataset", + b"define_dataset", + "define_flow", + b"define_flow", + "define_sql_graph_elements", + b"define_sql_graph_elements", + "drop_dataflow_graph", + b"drop_dataflow_graph", + "start_run", + b"start_run", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "command_type", + b"command_type", + "create_dataflow_graph", + b"create_dataflow_graph", + "define_dataset", + b"define_dataset", + "define_flow", + b"define_flow", + "define_sql_graph_elements", + b"define_sql_graph_elements", + "drop_dataflow_graph", + b"drop_dataflow_graph", + "start_run", + b"start_run", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["command_type", b"command_type"] + ) -> ( + typing_extensions.Literal[ + "create_dataflow_graph", + "define_dataset", + "define_flow", + "drop_dataflow_graph", + "start_run", + "define_sql_graph_elements", + ] + | None + ): ... + +global___PipelineCommand = PipelineCommand + +class DefineSqlGraphElements(google.protobuf.message.Message): + """Parses the SQL file and registers all datasets and flows.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + SQL_FILE_PATH_FIELD_NUMBER: builtins.int + SQL_TEXT_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The graph to attach this dataset to.""" + sql_file_path: builtins.str + """The full path to the SQL file. Can be relative or absolute.""" + sql_text: builtins.str + """The contents of the SQL file.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + sql_file_path: builtins.str | None = ..., + sql_text: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_sql_file_path", + b"_sql_file_path", + "_sql_text", + b"_sql_text", + "dataflow_graph_id", + b"dataflow_graph_id", + "sql_file_path", + b"sql_file_path", + "sql_text", + b"sql_text", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "_sql_file_path", + b"_sql_file_path", + "_sql_text", + b"_sql_text", + "dataflow_graph_id", + b"dataflow_graph_id", + "sql_file_path", + b"sql_file_path", + "sql_text", + b"sql_text", + ], + ) -> None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"] + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_sql_file_path", b"_sql_file_path"] + ) -> typing_extensions.Literal["sql_file_path"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_sql_text", b"_sql_text"] + ) -> typing_extensions.Literal["sql_text"] | None: ... + +global___DefineSqlGraphElements = DefineSqlGraphElements + +class PipelineCommandResult(google.protobuf.message.Message): + """Dispatch object for pipelines command results.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class CreateDataflowGraphResult(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATAFLOW_GRAPH_ID_FIELD_NUMBER: builtins.int + dataflow_graph_id: builtins.str + """The ID of the created graph.""" + def __init__( + self, + *, + dataflow_graph_id: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_dataflow_graph_id", + b"_dataflow_graph_id", + "dataflow_graph_id", + b"dataflow_graph_id", + ], + ) -> None: ... + def WhichOneof( + self, + oneof_group: typing_extensions.Literal["_dataflow_graph_id", b"_dataflow_graph_id"], + ) -> typing_extensions.Literal["dataflow_graph_id"] | None: ... + + CREATE_DATAFLOW_GRAPH_RESULT_FIELD_NUMBER: builtins.int + @property + def create_dataflow_graph_result( + self, + ) -> global___PipelineCommandResult.CreateDataflowGraphResult: ... + def __init__( + self, + *, + create_dataflow_graph_result: global___PipelineCommandResult.CreateDataflowGraphResult + | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "create_dataflow_graph_result", + b"create_dataflow_graph_result", + "result_type", + b"result_type", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "create_dataflow_graph_result", + b"create_dataflow_graph_result", + "result_type", + b"result_type", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["result_type", b"result_type"] + ) -> typing_extensions.Literal["create_dataflow_graph_result"] | None: ... + +global___PipelineCommandResult = PipelineCommandResult + +class PipelineEventResult(google.protobuf.message.Message): + """A response containing an event emitted during the run of a pipeline.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EVENT_FIELD_NUMBER: builtins.int + @property + def event(self) -> global___PipelineEvent: ... + def __init__( + self, + *, + event: global___PipelineEvent | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["event", b"event"] + ) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["event", b"event"]) -> None: ... + +global___PipelineEventResult = PipelineEventResult + +class PipelineEvent(google.protobuf.message.Message): + """An event emitted during the run of a graph.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TIMESTAMP_FIELD_NUMBER: builtins.int + MESSAGE_FIELD_NUMBER: builtins.int + timestamp: builtins.str + """The time of the event.""" + message: builtins.str + """The message that should be displayed to users.""" + def __init__( + self, + *, + timestamp: builtins.str | None = ..., + message: builtins.str | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_message", + b"_message", + "_timestamp", + b"_timestamp", + "message", + b"message", + "timestamp", + b"timestamp", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_message", + b"_message", + "_timestamp", + b"_timestamp", + "message", + b"message", + "timestamp", + b"timestamp", + ], + ) -> None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_message", b"_message"] + ) -> typing_extensions.Literal["message"] | None: ... + @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_timestamp", b"_timestamp"] + ) -> typing_extensions.Literal["timestamp"] | None: ... + +global___PipelineEvent = PipelineEvent