diff --git a/connector/connect/src/main/buf.gen.yaml b/connector/connect/src/main/buf.gen.yaml index 2a4b5df44dc3f..d74d08632fd94 100644 --- a/connector/connect/src/main/buf.gen.yaml +++ b/connector/connect/src/main/buf.gen.yaml @@ -27,7 +27,7 @@ plugins: - remote: buf.build/protocolbuffers/plugins/ruby:v21.2.0-1 out: gen/proto/ruby # Building the Python build and building the mypy interfaces. - - remote: buf.build/protocolbuffers/plugins/python:v3.20.0-1 + - remote: buf.build/protocolbuffers/plugins/python:v3.19.3-1 out: gen/proto/python - remote: buf.build/grpc/plugins/python:v1.47.0-1 out: gen/proto/python diff --git a/dev/requirements.txt b/dev/requirements.txt index 914c26b1fa148..d9497f63ad4b0 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -50,7 +50,7 @@ black==22.6.0 # Spark Connect (required) grpcio==1.48.1 -protobuf==4.21.6 +protobuf==3.19.4 # Spark Connect python proto generation plugin (optional) mypy-protobuf==3.3.0 diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index 0527e9b49aa86..e4055a28fd5e2 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -18,9 +18,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spark/connect/base.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -38,8 +39,135 @@ b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"\xc8\x02\n\x07Request\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0cuser_context\x18\x02 \x01(\x0b\x32".spark.connect.Request.UserContextR\x0buserContext\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x00R\nclientType\x88\x01\x01\x1az\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensionsB\x0e\n\x0c_client_type"\xe0\x06\n\x08Response\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32".spark.connect.Response.ArrowBatchH\x00R\narrowBatch\x12\x42\n\njson_batch\x18\x03 \x01(\x0b\x32!.spark.connect.Response.JSONBatchH\x00R\tjsonBatch\x12\x39\n\x07metrics\x18\x04 \x01(\x0b\x32\x1f.spark.connect.Response.MetricsR\x07metrics\x1a=\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x1a<\n\tJSONBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x1a\xe4\x03\n\x07Metrics\x12\x46\n\x07metrics\x18\x01 \x03(\x0b\x32,.spark.connect.Response.Metrics.MetricObjectR\x07metrics\x1a\xb6\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12o\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32\x42.spark.connect.Response.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1ap\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x41\n\x05value\x18\x02 \x01(\x0b\x32+.spark.connect.Response.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricTypeB\r\n\x0bresult_type"\x86\x01\n\x0f\x41nalyzeResponse\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12/\n\x06schema\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x12%\n\x0e\x65xplain_string\x18\x03 \x01(\tR\rexplainString2\xa2\x01\n\x13SparkConnectService\x12\x42\n\x0b\x45xecutePlan\x12\x16.spark.connect.Request\x1a\x17.spark.connect.Response"\x00\x30\x01\x12G\n\x0b\x41nalyzePlan\x12\x16.spark.connect.Request\x1a\x1e.spark.connect.AnalyzeResponse"\x00\x42"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.base_pb2", globals()) + +_PLAN = DESCRIPTOR.message_types_by_name["Plan"] +_REQUEST = DESCRIPTOR.message_types_by_name["Request"] +_REQUEST_USERCONTEXT = _REQUEST.nested_types_by_name["UserContext"] +_RESPONSE = DESCRIPTOR.message_types_by_name["Response"] +_RESPONSE_ARROWBATCH = _RESPONSE.nested_types_by_name["ArrowBatch"] +_RESPONSE_JSONBATCH = _RESPONSE.nested_types_by_name["JSONBatch"] +_RESPONSE_METRICS = _RESPONSE.nested_types_by_name["Metrics"] +_RESPONSE_METRICS_METRICOBJECT = _RESPONSE_METRICS.nested_types_by_name["MetricObject"] +_RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY = ( + _RESPONSE_METRICS_METRICOBJECT.nested_types_by_name["ExecutionMetricsEntry"] +) +_RESPONSE_METRICS_METRICVALUE = _RESPONSE_METRICS.nested_types_by_name["MetricValue"] +_ANALYZERESPONSE = DESCRIPTOR.message_types_by_name["AnalyzeResponse"] +Plan = _reflection.GeneratedProtocolMessageType( + "Plan", + (_message.Message,), + { + "DESCRIPTOR": _PLAN, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Plan) + }, +) +_sym_db.RegisterMessage(Plan) + +Request = _reflection.GeneratedProtocolMessageType( + "Request", + (_message.Message,), + { + "UserContext": _reflection.GeneratedProtocolMessageType( + "UserContext", + (_message.Message,), + { + "DESCRIPTOR": _REQUEST_USERCONTEXT, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Request.UserContext) + }, + ), + "DESCRIPTOR": _REQUEST, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Request) + }, +) +_sym_db.RegisterMessage(Request) +_sym_db.RegisterMessage(Request.UserContext) + +Response = _reflection.GeneratedProtocolMessageType( + "Response", + (_message.Message,), + { + "ArrowBatch": _reflection.GeneratedProtocolMessageType( + "ArrowBatch", + (_message.Message,), + { + "DESCRIPTOR": _RESPONSE_ARROWBATCH, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.ArrowBatch) + }, + ), + "JSONBatch": _reflection.GeneratedProtocolMessageType( + "JSONBatch", + (_message.Message,), + { + "DESCRIPTOR": _RESPONSE_JSONBATCH, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.JSONBatch) + }, + ), + "Metrics": _reflection.GeneratedProtocolMessageType( + "Metrics", + (_message.Message,), + { + "MetricObject": _reflection.GeneratedProtocolMessageType( + "MetricObject", + (_message.Message,), + { + "ExecutionMetricsEntry": _reflection.GeneratedProtocolMessageType( + "ExecutionMetricsEntry", + (_message.Message,), + { + "DESCRIPTOR": _RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.Metrics.MetricObject.ExecutionMetricsEntry) + }, + ), + "DESCRIPTOR": _RESPONSE_METRICS_METRICOBJECT, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.Metrics.MetricObject) + }, + ), + "MetricValue": _reflection.GeneratedProtocolMessageType( + "MetricValue", + (_message.Message,), + { + "DESCRIPTOR": _RESPONSE_METRICS_METRICVALUE, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.Metrics.MetricValue) + }, + ), + "DESCRIPTOR": _RESPONSE_METRICS, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response.Metrics) + }, + ), + "DESCRIPTOR": _RESPONSE, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Response) + }, +) +_sym_db.RegisterMessage(Response) +_sym_db.RegisterMessage(Response.ArrowBatch) +_sym_db.RegisterMessage(Response.JSONBatch) +_sym_db.RegisterMessage(Response.Metrics) +_sym_db.RegisterMessage(Response.Metrics.MetricObject) +_sym_db.RegisterMessage(Response.Metrics.MetricObject.ExecutionMetricsEntry) +_sym_db.RegisterMessage(Response.Metrics.MetricValue) + +AnalyzeResponse = _reflection.GeneratedProtocolMessageType( + "AnalyzeResponse", + (_message.Message,), + { + "DESCRIPTOR": _ANALYZERESPONSE, + "__module__": "spark.connect.base_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.AnalyzeResponse) + }, +) +_sym_db.RegisterMessage(AnalyzeResponse) + +_SPARKCONNECTSERVICE = DESCRIPTOR.services_by_name["SparkConnectService"] if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index 11f53322ce740..e7069a4b9a7a3 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -18,9 +18,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spark/connect/commands.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -36,8 +37,81 @@ b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\x94\x02\n\x07\x43ommand\x12N\n\x0f\x63reate_function\x18\x01 \x01(\x0b\x32#.spark.connect.CreateScalarFunctionH\x00R\x0e\x63reateFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeViewB\x0e\n\x0c\x63ommand_type"\x97\x04\n\x14\x43reateScalarFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12P\n\x08language\x18\x02 \x01(\x0e\x32\x34.spark.connect.CreateScalarFunction.FunctionLanguageR\x08language\x12\x1c\n\ttemporary\x18\x03 \x01(\x08R\ttemporary\x12>\n\x0e\x61rgument_types\x18\x04 \x03(\x0b\x32\x17.spark.connect.DataTypeR\rargumentTypes\x12\x38\n\x0breturn_type\x18\x05 \x01(\x0b\x32\x17.spark.connect.DataTypeR\nreturnType\x12\x31\n\x13serialized_function\x18\x06 \x01(\x0cH\x00R\x12serializedFunction\x12\'\n\x0eliteral_string\x18\x07 \x01(\tH\x00R\rliteralString"\x8b\x01\n\x10\x46unctionLanguage\x12!\n\x1d\x46UNCTION_LANGUAGE_UNSPECIFIED\x10\x00\x12\x19\n\x15\x46UNCTION_LANGUAGE_SQL\x10\x01\x12\x1c\n\x18\x46UNCTION_LANGUAGE_PYTHON\x10\x02\x12\x1b\n\x17\x46UNCTION_LANGUAGE_SCALA\x10\x03\x42\x15\n\x13\x66unction_definition"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\xe6\x05\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06source\x18\x02 \x01(\tR\x06source\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12\x1f\n\ntable_name\x18\x04 \x01(\tH\x00R\ttableName\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.commands_pb2", globals()) + +_COMMAND = DESCRIPTOR.message_types_by_name["Command"] +_CREATESCALARFUNCTION = DESCRIPTOR.message_types_by_name["CreateScalarFunction"] +_CREATEDATAFRAMEVIEWCOMMAND = DESCRIPTOR.message_types_by_name["CreateDataFrameViewCommand"] +_WRITEOPERATION = DESCRIPTOR.message_types_by_name["WriteOperation"] +_WRITEOPERATION_OPTIONSENTRY = _WRITEOPERATION.nested_types_by_name["OptionsEntry"] +_WRITEOPERATION_BUCKETBY = _WRITEOPERATION.nested_types_by_name["BucketBy"] +_CREATESCALARFUNCTION_FUNCTIONLANGUAGE = _CREATESCALARFUNCTION.enum_types_by_name[ + "FunctionLanguage" +] +_WRITEOPERATION_SAVEMODE = _WRITEOPERATION.enum_types_by_name["SaveMode"] +Command = _reflection.GeneratedProtocolMessageType( + "Command", + (_message.Message,), + { + "DESCRIPTOR": _COMMAND, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Command) + }, +) +_sym_db.RegisterMessage(Command) + +CreateScalarFunction = _reflection.GeneratedProtocolMessageType( + "CreateScalarFunction", + (_message.Message,), + { + "DESCRIPTOR": _CREATESCALARFUNCTION, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.CreateScalarFunction) + }, +) +_sym_db.RegisterMessage(CreateScalarFunction) + +CreateDataFrameViewCommand = _reflection.GeneratedProtocolMessageType( + "CreateDataFrameViewCommand", + (_message.Message,), + { + "DESCRIPTOR": _CREATEDATAFRAMEVIEWCOMMAND, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.CreateDataFrameViewCommand) + }, +) +_sym_db.RegisterMessage(CreateDataFrameViewCommand) + +WriteOperation = _reflection.GeneratedProtocolMessageType( + "WriteOperation", + (_message.Message,), + { + "OptionsEntry": _reflection.GeneratedProtocolMessageType( + "OptionsEntry", + (_message.Message,), + { + "DESCRIPTOR": _WRITEOPERATION_OPTIONSENTRY, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.WriteOperation.OptionsEntry) + }, + ), + "BucketBy": _reflection.GeneratedProtocolMessageType( + "BucketBy", + (_message.Message,), + { + "DESCRIPTOR": _WRITEOPERATION_BUCKETBY, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.WriteOperation.BucketBy) + }, + ), + "DESCRIPTOR": _WRITEOPERATION, + "__module__": "spark.connect.commands_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.WriteOperation) + }, +) +_sym_db.RegisterMessage(WriteOperation) +_sym_db.RegisterMessage(WriteOperation.OptionsEntry) +_sym_db.RegisterMessage(WriteOperation.BucketBy) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py index dca9d2cef47fe..c372df7d3247f 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.py +++ b/python/pyspark/sql/connect/proto/expressions_pb2.py @@ -18,9 +18,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spark/connect/expressions.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -36,8 +37,199 @@ b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19spark/connect/types.proto\x1a\x19google/protobuf/any.proto"\xf0\x17\n\nExpression\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x1a\xa3\x10\n\x07Literal\x12\x1a\n\x07\x62oolean\x18\x01 \x01(\x08H\x00R\x07\x62oolean\x12\x10\n\x02i8\x18\x02 \x01(\x05H\x00R\x02i8\x12\x12\n\x03i16\x18\x03 \x01(\x05H\x00R\x03i16\x12\x12\n\x03i32\x18\x05 \x01(\x05H\x00R\x03i32\x12\x12\n\x03i64\x18\x07 \x01(\x03H\x00R\x03i64\x12\x14\n\x04\x66p32\x18\n \x01(\x02H\x00R\x04\x66p32\x12\x14\n\x04\x66p64\x18\x0b \x01(\x01H\x00R\x04\x66p64\x12\x18\n\x06string\x18\x0c \x01(\tH\x00R\x06string\x12\x18\n\x06\x62inary\x18\r \x01(\x0cH\x00R\x06\x62inary\x12\x1e\n\ttimestamp\x18\x0e \x01(\x03H\x00R\ttimestamp\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x14\n\x04time\x18\x11 \x01(\x03H\x00R\x04time\x12l\n\x16interval_year_to_month\x18\x13 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalYearToMonthH\x00R\x13intervalYearToMonth\x12l\n\x16interval_day_to_second\x18\x14 \x01(\x0b\x32\x35.spark.connect.Expression.Literal.IntervalDayToSecondH\x00R\x13intervalDayToSecond\x12\x1f\n\nfixed_char\x18\x15 \x01(\tH\x00R\tfixedChar\x12\x46\n\x08var_char\x18\x16 \x01(\x0b\x32).spark.connect.Expression.Literal.VarCharH\x00R\x07varChar\x12#\n\x0c\x66ixed_binary\x18\x17 \x01(\x0cH\x00R\x0b\x66ixedBinary\x12\x45\n\x07\x64\x65\x63imal\x18\x18 \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x42\n\x06struct\x18\x19 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x12\x39\n\x03map\x18\x1a \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12#\n\x0ctimestamp_tz\x18\x1b \x01(\x03H\x00R\x0btimestampTz\x12\x14\n\x04uuid\x18\x1c \x01(\x0cH\x00R\x04uuid\x12-\n\x04null\x18\x1d \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12<\n\x04list\x18\x1e \x01(\x0b\x32&.spark.connect.Expression.Literal.ListH\x00R\x04list\x12=\n\nempty_list\x18\x1f \x01(\x0b\x32\x1c.spark.connect.DataType.ListH\x00R\temptyList\x12:\n\tempty_map\x18 \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x08\x65mptyMap\x12R\n\x0cuser_defined\x18! \x01(\x0b\x32-.spark.connect.Expression.Literal.UserDefinedH\x00R\x0buserDefined\x12\x1a\n\x08nullable\x18\x32 \x01(\x08R\x08nullable\x12\x38\n\x18type_variation_reference\x18\x33 \x01(\rR\x16typeVariationReference\x1a\x37\n\x07VarChar\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12\x16\n\x06length\x18\x02 \x01(\rR\x06length\x1aS\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value\x12\x1c\n\tprecision\x18\x02 \x01(\x05R\tprecision\x12\x14\n\x05scale\x18\x03 \x01(\x05R\x05scale\x1a\xce\x01\n\x03Map\x12M\n\nkey_values\x18\x01 \x03(\x0b\x32..spark.connect.Expression.Literal.Map.KeyValueR\tkeyValues\x1ax\n\x08KeyValue\x12\x33\n\x03key\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value\x1a\x43\n\x13IntervalYearToMonth\x12\x14\n\x05years\x18\x01 \x01(\x05R\x05years\x12\x16\n\x06months\x18\x02 \x01(\x05R\x06months\x1ag\n\x13IntervalDayToSecond\x12\x12\n\x04\x64\x61ys\x18\x01 \x01(\x05R\x04\x64\x61ys\x12\x18\n\x07seconds\x18\x02 \x01(\x05R\x07seconds\x12"\n\x0cmicroseconds\x18\x03 \x01(\x05R\x0cmicroseconds\x1a\x43\n\x06Struct\x12\x39\n\x06\x66ields\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06\x66ields\x1a\x41\n\x04List\x12\x39\n\x06values\x18\x01 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a`\n\x0bUserDefined\x12%\n\x0etype_reference\x18\x01 \x01(\rR\rtypeReference\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05valueB\x0e\n\x0cliteral_type\x1a\x46\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x1a\x63\n\x12UnresolvedFunction\x12\x14\n\x05parts\x18\x01 \x03(\tR\x05parts\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a\x10\n\x0eUnresolvedStar\x1aU\n\x12QualifiedAttribute\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12+\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x04type\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadataB\x0b\n\texpr_typeB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.expressions_pb2", globals()) + +_EXPRESSION = DESCRIPTOR.message_types_by_name["Expression"] +_EXPRESSION_LITERAL = _EXPRESSION.nested_types_by_name["Literal"] +_EXPRESSION_LITERAL_VARCHAR = _EXPRESSION_LITERAL.nested_types_by_name["VarChar"] +_EXPRESSION_LITERAL_DECIMAL = _EXPRESSION_LITERAL.nested_types_by_name["Decimal"] +_EXPRESSION_LITERAL_MAP = _EXPRESSION_LITERAL.nested_types_by_name["Map"] +_EXPRESSION_LITERAL_MAP_KEYVALUE = _EXPRESSION_LITERAL_MAP.nested_types_by_name["KeyValue"] +_EXPRESSION_LITERAL_INTERVALYEARTOMONTH = _EXPRESSION_LITERAL.nested_types_by_name[ + "IntervalYearToMonth" +] +_EXPRESSION_LITERAL_INTERVALDAYTOSECOND = _EXPRESSION_LITERAL.nested_types_by_name[ + "IntervalDayToSecond" +] +_EXPRESSION_LITERAL_STRUCT = _EXPRESSION_LITERAL.nested_types_by_name["Struct"] +_EXPRESSION_LITERAL_LIST = _EXPRESSION_LITERAL.nested_types_by_name["List"] +_EXPRESSION_LITERAL_USERDEFINED = _EXPRESSION_LITERAL.nested_types_by_name["UserDefined"] +_EXPRESSION_UNRESOLVEDATTRIBUTE = _EXPRESSION.nested_types_by_name["UnresolvedAttribute"] +_EXPRESSION_UNRESOLVEDFUNCTION = _EXPRESSION.nested_types_by_name["UnresolvedFunction"] +_EXPRESSION_EXPRESSIONSTRING = _EXPRESSION.nested_types_by_name["ExpressionString"] +_EXPRESSION_UNRESOLVEDSTAR = _EXPRESSION.nested_types_by_name["UnresolvedStar"] +_EXPRESSION_QUALIFIEDATTRIBUTE = _EXPRESSION.nested_types_by_name["QualifiedAttribute"] +_EXPRESSION_ALIAS = _EXPRESSION.nested_types_by_name["Alias"] +Expression = _reflection.GeneratedProtocolMessageType( + "Expression", + (_message.Message,), + { + "Literal": _reflection.GeneratedProtocolMessageType( + "Literal", + (_message.Message,), + { + "VarChar": _reflection.GeneratedProtocolMessageType( + "VarChar", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_VARCHAR, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.VarChar) + }, + ), + "Decimal": _reflection.GeneratedProtocolMessageType( + "Decimal", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_DECIMAL, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.Decimal) + }, + ), + "Map": _reflection.GeneratedProtocolMessageType( + "Map", + (_message.Message,), + { + "KeyValue": _reflection.GeneratedProtocolMessageType( + "KeyValue", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_MAP_KEYVALUE, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.Map.KeyValue) + }, + ), + "DESCRIPTOR": _EXPRESSION_LITERAL_MAP, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.Map) + }, + ), + "IntervalYearToMonth": _reflection.GeneratedProtocolMessageType( + "IntervalYearToMonth", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_INTERVALYEARTOMONTH, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.IntervalYearToMonth) + }, + ), + "IntervalDayToSecond": _reflection.GeneratedProtocolMessageType( + "IntervalDayToSecond", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_INTERVALDAYTOSECOND, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.IntervalDayToSecond) + }, + ), + "Struct": _reflection.GeneratedProtocolMessageType( + "Struct", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_STRUCT, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.Struct) + }, + ), + "List": _reflection.GeneratedProtocolMessageType( + "List", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_LIST, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.List) + }, + ), + "UserDefined": _reflection.GeneratedProtocolMessageType( + "UserDefined", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_LITERAL_USERDEFINED, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal.UserDefined) + }, + ), + "DESCRIPTOR": _EXPRESSION_LITERAL, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Literal) + }, + ), + "UnresolvedAttribute": _reflection.GeneratedProtocolMessageType( + "UnresolvedAttribute", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_UNRESOLVEDATTRIBUTE, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.UnresolvedAttribute) + }, + ), + "UnresolvedFunction": _reflection.GeneratedProtocolMessageType( + "UnresolvedFunction", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_UNRESOLVEDFUNCTION, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.UnresolvedFunction) + }, + ), + "ExpressionString": _reflection.GeneratedProtocolMessageType( + "ExpressionString", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_EXPRESSIONSTRING, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.ExpressionString) + }, + ), + "UnresolvedStar": _reflection.GeneratedProtocolMessageType( + "UnresolvedStar", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_UNRESOLVEDSTAR, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.UnresolvedStar) + }, + ), + "QualifiedAttribute": _reflection.GeneratedProtocolMessageType( + "QualifiedAttribute", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_QUALIFIEDATTRIBUTE, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.QualifiedAttribute) + }, + ), + "Alias": _reflection.GeneratedProtocolMessageType( + "Alias", + (_message.Message,), + { + "DESCRIPTOR": _EXPRESSION_ALIAS, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression.Alias) + }, + ), + "DESCRIPTOR": _EXPRESSION, + "__module__": "spark.connect.expressions_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Expression) + }, +) +_sym_db.RegisterMessage(Expression) +_sym_db.RegisterMessage(Expression.Literal) +_sym_db.RegisterMessage(Expression.Literal.VarChar) +_sym_db.RegisterMessage(Expression.Literal.Decimal) +_sym_db.RegisterMessage(Expression.Literal.Map) +_sym_db.RegisterMessage(Expression.Literal.Map.KeyValue) +_sym_db.RegisterMessage(Expression.Literal.IntervalYearToMonth) +_sym_db.RegisterMessage(Expression.Literal.IntervalDayToSecond) +_sym_db.RegisterMessage(Expression.Literal.Struct) +_sym_db.RegisterMessage(Expression.Literal.List) +_sym_db.RegisterMessage(Expression.Literal.UserDefined) +_sym_db.RegisterMessage(Expression.UnresolvedAttribute) +_sym_db.RegisterMessage(Expression.UnresolvedFunction) +_sym_db.RegisterMessage(Expression.ExpressionString) +_sym_db.RegisterMessage(Expression.UnresolvedStar) +_sym_db.RegisterMessage(Expression.QualifiedAttribute) +_sym_db.RegisterMessage(Expression.Alias) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py index 9bc5e75ea6460..c586e4bdedaf0 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.py +++ b/python/pyspark/sql/connect/proto/relations_pb2.py @@ -18,9 +18,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spark/connect/relations.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -35,8 +36,368 @@ b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa6\x0b\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12\x34\n\x06set_op\x18\x06 \x01(\x0b\x32\x1b.spark.connect.SetOperationH\x00R\x05setOp\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12,\n\x05range\x18\x0f \x01(\x0b\x32\x14.spark.connect.RangeH\x00R\x05range\x12\x45\n\x0esubquery_alias\x18\x10 \x01(\x0b\x32\x1c.spark.connect.SubqueryAliasH\x00R\rsubqueryAlias\x12>\n\x0brepartition\x18\x11 \x01(\x0b\x32\x1a.spark.connect.RepartitionH\x00R\x0brepartition\x12|\n#rename_columns_by_same_length_names\x18\x12 \x01(\x0b\x32-.spark.connect.RenameColumnsBySameLengthNamesH\x00R\x1erenameColumnsBySameLengthNames\x12w\n"rename_columns_by_name_to_name_map\x18\x13 \x01(\x0b\x32+.spark.connect.RenameColumnsByNameToNameMapH\x00R\x1crenameColumnsByNameToNameMap\x12<\n\x0bshow_string\x18\x14 \x01(\x0b\x32\x19.spark.connect.ShowStringH\x00R\nshowString\x12\x30\n\x07\x66ill_na\x18Z \x01(\x0b\x32\x15.spark.connect.NAFillH\x00R\x06\x66illNa\x12\x36\n\x07summary\x18\x64 \x01(\x0b\x32\x1a.spark.connect.StatSummaryH\x00R\x07summary\x12\x39\n\x08\x63rosstab\x18\x65 \x01(\x0b\x32\x1b.spark.connect.StatCrosstabH\x00R\x08\x63rosstab\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"1\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo"\x1b\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query"\xaa\x03\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x1a=\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x1a\xcf\x01\n\nDataSource\x12\x16\n\x06\x66ormat\x18\x01 \x01(\tR\x06\x66ormat\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x00R\x06schema\x88\x01\x01\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07_schemaB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\xc2\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType\x12#\n\rusing_columns\x18\x05 \x03(\tR\x0cusingColumns"\xbb\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06"\x8c\x03\n\x0cSetOperation\x12\x36\n\nleft_input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\tleftInput\x12\x38\n\x0bright_input\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\nrightInput\x12\x45\n\x0bset_op_type\x18\x03 \x01(\x0e\x32%.spark.connect.SetOperation.SetOpTypeR\tsetOpType\x12\x1a\n\x06is_all\x18\x04 \x01(\x08H\x00R\x05isAll\x88\x01\x01\x12\x1c\n\x07\x62y_name\x18\x05 \x01(\x08H\x01R\x06\x62yName\x88\x01\x01"r\n\tSetOpType\x12\x1b\n\x17SET_OP_TYPE_UNSPECIFIED\x10\x00\x12\x19\n\x15SET_OP_TYPE_INTERSECT\x10\x01\x12\x15\n\x11SET_OP_TYPE_UNION\x10\x02\x12\x16\n\x12SET_OP_TYPE_EXCEPT\x10\x03\x42\t\n\x07_is_allB\n\n\x08_by_name"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"\xd2\x01\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12H\n\x12result_expressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x11resultExpressions"\xa6\x04\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\x0bsort_fields\x18\x02 \x03(\x0b\x32\x1d.spark.connect.Sort.SortFieldR\nsortFields\x12 \n\tis_global\x18\x03 \x01(\x08H\x00R\x08isGlobal\x88\x01\x01\x1a\xbc\x01\n\tSortField\x12\x39\n\nexpression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nexpression\x12?\n\tdirection\x18\x02 \x01(\x0e\x32!.spark.connect.Sort.SortDirectionR\tdirection\x12\x33\n\x05nulls\x18\x03 \x01(\x0e\x32\x1d.spark.connect.Sort.SortNullsR\x05nulls"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"R\n\tSortNulls\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x42\x0c\n\n_is_global"\xab\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12\x32\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08H\x00R\x10\x61llColumnsAsKeys\x88\x01\x01\x42\x16\n\x14_all_columns_as_keys"]\n\rLocalRelation\x12L\n\nattributes\x18\x01 \x03(\x0b\x32,.spark.connect.Expression.QualifiedAttributeR\nattributes"\xe0\x01\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12.\n\x10with_replacement\x18\x04 \x01(\x08H\x00R\x0fwithReplacement\x88\x01\x01\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x01R\x04seed\x88\x01\x01\x42\x13\n\x11_with_replacementB\x07\n\x05_seed"\x91\x01\n\x05Range\x12\x19\n\x05start\x18\x01 \x01(\x03H\x00R\x05start\x88\x01\x01\x12\x10\n\x03\x65nd\x18\x02 \x01(\x03R\x03\x65nd\x12\x12\n\x04step\x18\x03 \x01(\x03R\x04step\x12*\n\x0enum_partitions\x18\x04 \x01(\x05H\x01R\rnumPartitions\x88\x01\x01\x42\x08\n\x06_startB\x11\n\x0f_num_partitions"r\n\rSubqueryAlias\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\x12\x1c\n\tqualifier\x18\x03 \x03(\tR\tqualifier"\x8e\x01\n\x0bRepartition\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12%\n\x0enum_partitions\x18\x02 \x01(\x05R\rnumPartitions\x12\x1d\n\x07shuffle\x18\x03 \x01(\x08H\x00R\x07shuffle\x88\x01\x01\x42\n\n\x08_shuffle"\x8d\x01\n\nShowString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x18\n\x07numRows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate\x12\x1a\n\x08vertical\x18\x04 \x01(\x08R\x08vertical"\\\n\x0bStatSummary\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1e\n\nstatistics\x18\x02 \x03(\tR\nstatistics"e\n\x0cStatCrosstab\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"\x86\x01\n\x06NAFill\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x39\n\x06values\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"r\n\x1eRenameColumnsBySameLengthNames\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames"\x83\x02\n\x1cRenameColumnsByNameToNameMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12o\n\x12rename_columns_map\x18\x02 \x03(\x0b\x32\x41.spark.connect.RenameColumnsByNameToNameMap.RenameColumnsMapEntryR\x10renameColumnsMap\x1a\x43\n\x15RenameColumnsMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.relations_pb2", globals()) + +_RELATION = DESCRIPTOR.message_types_by_name["Relation"] +_UNKNOWN = DESCRIPTOR.message_types_by_name["Unknown"] +_RELATIONCOMMON = DESCRIPTOR.message_types_by_name["RelationCommon"] +_SQL = DESCRIPTOR.message_types_by_name["SQL"] +_READ = DESCRIPTOR.message_types_by_name["Read"] +_READ_NAMEDTABLE = _READ.nested_types_by_name["NamedTable"] +_READ_DATASOURCE = _READ.nested_types_by_name["DataSource"] +_READ_DATASOURCE_OPTIONSENTRY = _READ_DATASOURCE.nested_types_by_name["OptionsEntry"] +_PROJECT = DESCRIPTOR.message_types_by_name["Project"] +_FILTER = DESCRIPTOR.message_types_by_name["Filter"] +_JOIN = DESCRIPTOR.message_types_by_name["Join"] +_SETOPERATION = DESCRIPTOR.message_types_by_name["SetOperation"] +_LIMIT = DESCRIPTOR.message_types_by_name["Limit"] +_OFFSET = DESCRIPTOR.message_types_by_name["Offset"] +_AGGREGATE = DESCRIPTOR.message_types_by_name["Aggregate"] +_SORT = DESCRIPTOR.message_types_by_name["Sort"] +_SORT_SORTFIELD = _SORT.nested_types_by_name["SortField"] +_DEDUPLICATE = DESCRIPTOR.message_types_by_name["Deduplicate"] +_LOCALRELATION = DESCRIPTOR.message_types_by_name["LocalRelation"] +_SAMPLE = DESCRIPTOR.message_types_by_name["Sample"] +_RANGE = DESCRIPTOR.message_types_by_name["Range"] +_SUBQUERYALIAS = DESCRIPTOR.message_types_by_name["SubqueryAlias"] +_REPARTITION = DESCRIPTOR.message_types_by_name["Repartition"] +_SHOWSTRING = DESCRIPTOR.message_types_by_name["ShowString"] +_STATSUMMARY = DESCRIPTOR.message_types_by_name["StatSummary"] +_STATCROSSTAB = DESCRIPTOR.message_types_by_name["StatCrosstab"] +_NAFILL = DESCRIPTOR.message_types_by_name["NAFill"] +_RENAMECOLUMNSBYSAMELENGTHNAMES = DESCRIPTOR.message_types_by_name["RenameColumnsBySameLengthNames"] +_RENAMECOLUMNSBYNAMETONAMEMAP = DESCRIPTOR.message_types_by_name["RenameColumnsByNameToNameMap"] +_RENAMECOLUMNSBYNAMETONAMEMAP_RENAMECOLUMNSMAPENTRY = ( + _RENAMECOLUMNSBYNAMETONAMEMAP.nested_types_by_name["RenameColumnsMapEntry"] +) +_JOIN_JOINTYPE = _JOIN.enum_types_by_name["JoinType"] +_SETOPERATION_SETOPTYPE = _SETOPERATION.enum_types_by_name["SetOpType"] +_SORT_SORTDIRECTION = _SORT.enum_types_by_name["SortDirection"] +_SORT_SORTNULLS = _SORT.enum_types_by_name["SortNulls"] +Relation = _reflection.GeneratedProtocolMessageType( + "Relation", + (_message.Message,), + { + "DESCRIPTOR": _RELATION, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Relation) + }, +) +_sym_db.RegisterMessage(Relation) + +Unknown = _reflection.GeneratedProtocolMessageType( + "Unknown", + (_message.Message,), + { + "DESCRIPTOR": _UNKNOWN, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Unknown) + }, +) +_sym_db.RegisterMessage(Unknown) + +RelationCommon = _reflection.GeneratedProtocolMessageType( + "RelationCommon", + (_message.Message,), + { + "DESCRIPTOR": _RELATIONCOMMON, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.RelationCommon) + }, +) +_sym_db.RegisterMessage(RelationCommon) + +SQL = _reflection.GeneratedProtocolMessageType( + "SQL", + (_message.Message,), + { + "DESCRIPTOR": _SQL, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.SQL) + }, +) +_sym_db.RegisterMessage(SQL) + +Read = _reflection.GeneratedProtocolMessageType( + "Read", + (_message.Message,), + { + "NamedTable": _reflection.GeneratedProtocolMessageType( + "NamedTable", + (_message.Message,), + { + "DESCRIPTOR": _READ_NAMEDTABLE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Read.NamedTable) + }, + ), + "DataSource": _reflection.GeneratedProtocolMessageType( + "DataSource", + (_message.Message,), + { + "OptionsEntry": _reflection.GeneratedProtocolMessageType( + "OptionsEntry", + (_message.Message,), + { + "DESCRIPTOR": _READ_DATASOURCE_OPTIONSENTRY, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Read.DataSource.OptionsEntry) + }, + ), + "DESCRIPTOR": _READ_DATASOURCE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Read.DataSource) + }, + ), + "DESCRIPTOR": _READ, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Read) + }, +) +_sym_db.RegisterMessage(Read) +_sym_db.RegisterMessage(Read.NamedTable) +_sym_db.RegisterMessage(Read.DataSource) +_sym_db.RegisterMessage(Read.DataSource.OptionsEntry) + +Project = _reflection.GeneratedProtocolMessageType( + "Project", + (_message.Message,), + { + "DESCRIPTOR": _PROJECT, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Project) + }, +) +_sym_db.RegisterMessage(Project) + +Filter = _reflection.GeneratedProtocolMessageType( + "Filter", + (_message.Message,), + { + "DESCRIPTOR": _FILTER, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Filter) + }, +) +_sym_db.RegisterMessage(Filter) + +Join = _reflection.GeneratedProtocolMessageType( + "Join", + (_message.Message,), + { + "DESCRIPTOR": _JOIN, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Join) + }, +) +_sym_db.RegisterMessage(Join) + +SetOperation = _reflection.GeneratedProtocolMessageType( + "SetOperation", + (_message.Message,), + { + "DESCRIPTOR": _SETOPERATION, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.SetOperation) + }, +) +_sym_db.RegisterMessage(SetOperation) + +Limit = _reflection.GeneratedProtocolMessageType( + "Limit", + (_message.Message,), + { + "DESCRIPTOR": _LIMIT, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Limit) + }, +) +_sym_db.RegisterMessage(Limit) + +Offset = _reflection.GeneratedProtocolMessageType( + "Offset", + (_message.Message,), + { + "DESCRIPTOR": _OFFSET, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Offset) + }, +) +_sym_db.RegisterMessage(Offset) + +Aggregate = _reflection.GeneratedProtocolMessageType( + "Aggregate", + (_message.Message,), + { + "DESCRIPTOR": _AGGREGATE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Aggregate) + }, +) +_sym_db.RegisterMessage(Aggregate) + +Sort = _reflection.GeneratedProtocolMessageType( + "Sort", + (_message.Message,), + { + "SortField": _reflection.GeneratedProtocolMessageType( + "SortField", + (_message.Message,), + { + "DESCRIPTOR": _SORT_SORTFIELD, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Sort.SortField) + }, + ), + "DESCRIPTOR": _SORT, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Sort) + }, +) +_sym_db.RegisterMessage(Sort) +_sym_db.RegisterMessage(Sort.SortField) + +Deduplicate = _reflection.GeneratedProtocolMessageType( + "Deduplicate", + (_message.Message,), + { + "DESCRIPTOR": _DEDUPLICATE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Deduplicate) + }, +) +_sym_db.RegisterMessage(Deduplicate) + +LocalRelation = _reflection.GeneratedProtocolMessageType( + "LocalRelation", + (_message.Message,), + { + "DESCRIPTOR": _LOCALRELATION, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.LocalRelation) + }, +) +_sym_db.RegisterMessage(LocalRelation) + +Sample = _reflection.GeneratedProtocolMessageType( + "Sample", + (_message.Message,), + { + "DESCRIPTOR": _SAMPLE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Sample) + }, +) +_sym_db.RegisterMessage(Sample) + +Range = _reflection.GeneratedProtocolMessageType( + "Range", + (_message.Message,), + { + "DESCRIPTOR": _RANGE, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Range) + }, +) +_sym_db.RegisterMessage(Range) + +SubqueryAlias = _reflection.GeneratedProtocolMessageType( + "SubqueryAlias", + (_message.Message,), + { + "DESCRIPTOR": _SUBQUERYALIAS, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.SubqueryAlias) + }, +) +_sym_db.RegisterMessage(SubqueryAlias) + +Repartition = _reflection.GeneratedProtocolMessageType( + "Repartition", + (_message.Message,), + { + "DESCRIPTOR": _REPARTITION, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.Repartition) + }, +) +_sym_db.RegisterMessage(Repartition) + +ShowString = _reflection.GeneratedProtocolMessageType( + "ShowString", + (_message.Message,), + { + "DESCRIPTOR": _SHOWSTRING, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.ShowString) + }, +) +_sym_db.RegisterMessage(ShowString) + +StatSummary = _reflection.GeneratedProtocolMessageType( + "StatSummary", + (_message.Message,), + { + "DESCRIPTOR": _STATSUMMARY, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.StatSummary) + }, +) +_sym_db.RegisterMessage(StatSummary) + +StatCrosstab = _reflection.GeneratedProtocolMessageType( + "StatCrosstab", + (_message.Message,), + { + "DESCRIPTOR": _STATCROSSTAB, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.StatCrosstab) + }, +) +_sym_db.RegisterMessage(StatCrosstab) + +NAFill = _reflection.GeneratedProtocolMessageType( + "NAFill", + (_message.Message,), + { + "DESCRIPTOR": _NAFILL, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.NAFill) + }, +) +_sym_db.RegisterMessage(NAFill) + +RenameColumnsBySameLengthNames = _reflection.GeneratedProtocolMessageType( + "RenameColumnsBySameLengthNames", + (_message.Message,), + { + "DESCRIPTOR": _RENAMECOLUMNSBYSAMELENGTHNAMES, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.RenameColumnsBySameLengthNames) + }, +) +_sym_db.RegisterMessage(RenameColumnsBySameLengthNames) + +RenameColumnsByNameToNameMap = _reflection.GeneratedProtocolMessageType( + "RenameColumnsByNameToNameMap", + (_message.Message,), + { + "RenameColumnsMapEntry": _reflection.GeneratedProtocolMessageType( + "RenameColumnsMapEntry", + (_message.Message,), + { + "DESCRIPTOR": _RENAMECOLUMNSBYNAMETONAMEMAP_RENAMECOLUMNSMAPENTRY, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.RenameColumnsByNameToNameMap.RenameColumnsMapEntry) + }, + ), + "DESCRIPTOR": _RENAMECOLUMNSBYNAMETONAMEMAP, + "__module__": "spark.connect.relations_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.RenameColumnsByNameToNameMap) + }, +) +_sym_db.RegisterMessage(RenameColumnsByNameToNameMap) +_sym_db.RegisterMessage(RenameColumnsByNameToNameMap.RenameColumnsMapEntry) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None diff --git a/python/pyspark/sql/connect/proto/types_pb2.py b/python/pyspark/sql/connect/proto/types_pb2.py index e904443714289..3507b03602ca0 100644 --- a/python/pyspark/sql/connect/proto/types_pb2.py +++ b/python/pyspark/sql/connect/proto/types_pb2.py @@ -18,9 +18,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: spark/connect/types.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -32,8 +33,294 @@ b'\n\x19spark/connect/types.proto\x12\rspark.connect"\xc1\x1c\n\x08\x44\x61taType\x12\x35\n\x04\x62ool\x18\x01 \x01(\x0b\x32\x1f.spark.connect.DataType.BooleanH\x00R\x04\x62ool\x12,\n\x02i8\x18\x02 \x01(\x0b\x32\x1a.spark.connect.DataType.I8H\x00R\x02i8\x12/\n\x03i16\x18\x03 \x01(\x0b\x32\x1b.spark.connect.DataType.I16H\x00R\x03i16\x12/\n\x03i32\x18\x05 \x01(\x0b\x32\x1b.spark.connect.DataType.I32H\x00R\x03i32\x12/\n\x03i64\x18\x07 \x01(\x0b\x32\x1b.spark.connect.DataType.I64H\x00R\x03i64\x12\x32\n\x04\x66p32\x18\n \x01(\x0b\x32\x1c.spark.connect.DataType.FP32H\x00R\x04\x66p32\x12\x32\n\x04\x66p64\x18\x0b \x01(\x0b\x32\x1c.spark.connect.DataType.FP64H\x00R\x04\x66p64\x12\x38\n\x06string\x18\x0c \x01(\x0b\x32\x1e.spark.connect.DataType.StringH\x00R\x06string\x12\x38\n\x06\x62inary\x18\r \x01(\x0b\x32\x1e.spark.connect.DataType.BinaryH\x00R\x06\x62inary\x12\x41\n\ttimestamp\x18\x0e \x01(\x0b\x32!.spark.connect.DataType.TimestampH\x00R\ttimestamp\x12\x32\n\x04\x64\x61te\x18\x10 \x01(\x0b\x32\x1c.spark.connect.DataType.DateH\x00R\x04\x64\x61te\x12\x32\n\x04time\x18\x11 \x01(\x0b\x32\x1c.spark.connect.DataType.TimeH\x00R\x04time\x12K\n\rinterval_year\x18\x13 \x01(\x0b\x32$.spark.connect.DataType.IntervalYearH\x00R\x0cintervalYear\x12H\n\x0cinterval_day\x18\x14 \x01(\x0b\x32#.spark.connect.DataType.IntervalDayH\x00R\x0bintervalDay\x12H\n\x0ctimestamp_tz\x18\x1d \x01(\x0b\x32#.spark.connect.DataType.TimestampTZH\x00R\x0btimestampTz\x12\x32\n\x04uuid\x18 \x01(\x0b\x32\x1c.spark.connect.DataType.UUIDH\x00R\x04uuid\x12\x42\n\nfixed_char\x18\x15 \x01(\x0b\x32!.spark.connect.DataType.FixedCharH\x00R\tfixedChar\x12;\n\x07varchar\x18\x16 \x01(\x0b\x32\x1f.spark.connect.DataType.VarCharH\x00R\x07varchar\x12H\n\x0c\x66ixed_binary\x18\x17 \x01(\x0b\x32#.spark.connect.DataType.FixedBinaryH\x00R\x0b\x66ixedBinary\x12;\n\x07\x64\x65\x63imal\x18\x18 \x01(\x0b\x32\x1f.spark.connect.DataType.DecimalH\x00R\x07\x64\x65\x63imal\x12\x38\n\x06struct\x18\x19 \x01(\x0b\x32\x1e.spark.connect.DataType.StructH\x00R\x06struct\x12\x32\n\x04list\x18\x1b \x01(\x0b\x32\x1c.spark.connect.DataType.ListH\x00R\x04list\x12/\n\x03map\x18\x1c \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x03map\x12?\n\x1buser_defined_type_reference\x18\x1f \x01(\rH\x00R\x18userDefinedTypeReference\x1a\x43\n\x07\x42oolean\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a>\n\x02I8\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a?\n\x03I16\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a?\n\x03I32\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a?\n\x03I64\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x46P32\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x46P64\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x42\n\x06String\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x42\n\x06\x42inary\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x45\n\tTimestamp\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x44\x61te\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04Time\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aG\n\x0bTimestampTZ\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aH\n\x0cIntervalYear\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aG\n\x0bIntervalDay\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04UUID\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a]\n\tFixedChar\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a[\n\x07VarChar\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a_\n\x0b\x46ixedBinary\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1aw\n\x07\x44\x65\x63imal\x12\x14\n\x05scale\x18\x01 \x01(\x05R\x05scale\x12\x1c\n\tprecision\x18\x02 \x01(\x05R\tprecision\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReference\x1a\xf6\x01\n\x0bStructField\x12+\n\x04type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x04type\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1a\n\x08nullable\x18\x03 \x01(\x08R\x08nullable\x12M\n\x08metadata\x18\x04 \x03(\x0b\x32\x31.spark.connect.DataType.StructField.MetadataEntryR\x08metadata\x1a;\n\rMetadataEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x7f\n\x06Struct\x12;\n\x06\x66ields\x18\x01 \x03(\x0b\x32#.spark.connect.DataType.StructFieldR\x06\x66ields\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\xa0\x01\n\x04List\x12\x33\n\x08\x44\x61taType\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x44\x61taType\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x12)\n\x10\x65lement_nullable\x18\x03 \x01(\x08R\x0f\x65lementNullable\x1a\xc0\x01\n\x03Map\x12)\n\x03key\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x03key\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x05value\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReference\x12%\n\x0evalue_nullable\x18\x04 \x01(\x08R\rvalueNullableB\x06\n\x04kindB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "spark.connect.types_pb2", globals()) + +_DATATYPE = DESCRIPTOR.message_types_by_name["DataType"] +_DATATYPE_BOOLEAN = _DATATYPE.nested_types_by_name["Boolean"] +_DATATYPE_I8 = _DATATYPE.nested_types_by_name["I8"] +_DATATYPE_I16 = _DATATYPE.nested_types_by_name["I16"] +_DATATYPE_I32 = _DATATYPE.nested_types_by_name["I32"] +_DATATYPE_I64 = _DATATYPE.nested_types_by_name["I64"] +_DATATYPE_FP32 = _DATATYPE.nested_types_by_name["FP32"] +_DATATYPE_FP64 = _DATATYPE.nested_types_by_name["FP64"] +_DATATYPE_STRING = _DATATYPE.nested_types_by_name["String"] +_DATATYPE_BINARY = _DATATYPE.nested_types_by_name["Binary"] +_DATATYPE_TIMESTAMP = _DATATYPE.nested_types_by_name["Timestamp"] +_DATATYPE_DATE = _DATATYPE.nested_types_by_name["Date"] +_DATATYPE_TIME = _DATATYPE.nested_types_by_name["Time"] +_DATATYPE_TIMESTAMPTZ = _DATATYPE.nested_types_by_name["TimestampTZ"] +_DATATYPE_INTERVALYEAR = _DATATYPE.nested_types_by_name["IntervalYear"] +_DATATYPE_INTERVALDAY = _DATATYPE.nested_types_by_name["IntervalDay"] +_DATATYPE_UUID = _DATATYPE.nested_types_by_name["UUID"] +_DATATYPE_FIXEDCHAR = _DATATYPE.nested_types_by_name["FixedChar"] +_DATATYPE_VARCHAR = _DATATYPE.nested_types_by_name["VarChar"] +_DATATYPE_FIXEDBINARY = _DATATYPE.nested_types_by_name["FixedBinary"] +_DATATYPE_DECIMAL = _DATATYPE.nested_types_by_name["Decimal"] +_DATATYPE_STRUCTFIELD = _DATATYPE.nested_types_by_name["StructField"] +_DATATYPE_STRUCTFIELD_METADATAENTRY = _DATATYPE_STRUCTFIELD.nested_types_by_name["MetadataEntry"] +_DATATYPE_STRUCT = _DATATYPE.nested_types_by_name["Struct"] +_DATATYPE_LIST = _DATATYPE.nested_types_by_name["List"] +_DATATYPE_MAP = _DATATYPE.nested_types_by_name["Map"] +DataType = _reflection.GeneratedProtocolMessageType( + "DataType", + (_message.Message,), + { + "Boolean": _reflection.GeneratedProtocolMessageType( + "Boolean", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_BOOLEAN, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Boolean) + }, + ), + "I8": _reflection.GeneratedProtocolMessageType( + "I8", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_I8, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.I8) + }, + ), + "I16": _reflection.GeneratedProtocolMessageType( + "I16", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_I16, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.I16) + }, + ), + "I32": _reflection.GeneratedProtocolMessageType( + "I32", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_I32, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.I32) + }, + ), + "I64": _reflection.GeneratedProtocolMessageType( + "I64", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_I64, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.I64) + }, + ), + "FP32": _reflection.GeneratedProtocolMessageType( + "FP32", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_FP32, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.FP32) + }, + ), + "FP64": _reflection.GeneratedProtocolMessageType( + "FP64", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_FP64, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.FP64) + }, + ), + "String": _reflection.GeneratedProtocolMessageType( + "String", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_STRING, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.String) + }, + ), + "Binary": _reflection.GeneratedProtocolMessageType( + "Binary", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_BINARY, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Binary) + }, + ), + "Timestamp": _reflection.GeneratedProtocolMessageType( + "Timestamp", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_TIMESTAMP, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Timestamp) + }, + ), + "Date": _reflection.GeneratedProtocolMessageType( + "Date", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_DATE, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Date) + }, + ), + "Time": _reflection.GeneratedProtocolMessageType( + "Time", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_TIME, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Time) + }, + ), + "TimestampTZ": _reflection.GeneratedProtocolMessageType( + "TimestampTZ", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_TIMESTAMPTZ, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.TimestampTZ) + }, + ), + "IntervalYear": _reflection.GeneratedProtocolMessageType( + "IntervalYear", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_INTERVALYEAR, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.IntervalYear) + }, + ), + "IntervalDay": _reflection.GeneratedProtocolMessageType( + "IntervalDay", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_INTERVALDAY, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.IntervalDay) + }, + ), + "UUID": _reflection.GeneratedProtocolMessageType( + "UUID", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_UUID, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.UUID) + }, + ), + "FixedChar": _reflection.GeneratedProtocolMessageType( + "FixedChar", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_FIXEDCHAR, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.FixedChar) + }, + ), + "VarChar": _reflection.GeneratedProtocolMessageType( + "VarChar", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_VARCHAR, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.VarChar) + }, + ), + "FixedBinary": _reflection.GeneratedProtocolMessageType( + "FixedBinary", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_FIXEDBINARY, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.FixedBinary) + }, + ), + "Decimal": _reflection.GeneratedProtocolMessageType( + "Decimal", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_DECIMAL, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Decimal) + }, + ), + "StructField": _reflection.GeneratedProtocolMessageType( + "StructField", + (_message.Message,), + { + "MetadataEntry": _reflection.GeneratedProtocolMessageType( + "MetadataEntry", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_STRUCTFIELD_METADATAENTRY, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.StructField.MetadataEntry) + }, + ), + "DESCRIPTOR": _DATATYPE_STRUCTFIELD, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.StructField) + }, + ), + "Struct": _reflection.GeneratedProtocolMessageType( + "Struct", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_STRUCT, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Struct) + }, + ), + "List": _reflection.GeneratedProtocolMessageType( + "List", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_LIST, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.List) + }, + ), + "Map": _reflection.GeneratedProtocolMessageType( + "Map", + (_message.Message,), + { + "DESCRIPTOR": _DATATYPE_MAP, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType.Map) + }, + ), + "DESCRIPTOR": _DATATYPE, + "__module__": "spark.connect.types_pb2" + # @@protoc_insertion_point(class_scope:spark.connect.DataType) + }, +) +_sym_db.RegisterMessage(DataType) +_sym_db.RegisterMessage(DataType.Boolean) +_sym_db.RegisterMessage(DataType.I8) +_sym_db.RegisterMessage(DataType.I16) +_sym_db.RegisterMessage(DataType.I32) +_sym_db.RegisterMessage(DataType.I64) +_sym_db.RegisterMessage(DataType.FP32) +_sym_db.RegisterMessage(DataType.FP64) +_sym_db.RegisterMessage(DataType.String) +_sym_db.RegisterMessage(DataType.Binary) +_sym_db.RegisterMessage(DataType.Timestamp) +_sym_db.RegisterMessage(DataType.Date) +_sym_db.RegisterMessage(DataType.Time) +_sym_db.RegisterMessage(DataType.TimestampTZ) +_sym_db.RegisterMessage(DataType.IntervalYear) +_sym_db.RegisterMessage(DataType.IntervalDay) +_sym_db.RegisterMessage(DataType.UUID) +_sym_db.RegisterMessage(DataType.FixedChar) +_sym_db.RegisterMessage(DataType.VarChar) +_sym_db.RegisterMessage(DataType.FixedBinary) +_sym_db.RegisterMessage(DataType.Decimal) +_sym_db.RegisterMessage(DataType.StructField) +_sym_db.RegisterMessage(DataType.StructField.MetadataEntry) +_sym_db.RegisterMessage(DataType.Struct) +_sym_db.RegisterMessage(DataType.List) +_sym_db.RegisterMessage(DataType.Map) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None