Protobuf generate V2 and V3 protos and extend tests. #38324
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| syntax = "proto2"; | ||
|
|
||
| package org.apache.spark.sql.protobuf.protos.v2; | ||
| option java_outer_classname = "CatalystTypes"; | ||
|
|
||
| // TODO: import one or more protobuf files. | ||
|
|
||
| message BooleanMsg { | ||
| optional bool bool_type = 1; | ||
| } | ||
| message IntegerMsg { | ||
| optional int32 int32_type = 1; | ||
| } | ||
| message DoubleMsg { | ||
| optional double double_type = 1; | ||
| } | ||
| message FloatMsg { | ||
| optional float float_type = 1; | ||
| } | ||
| message BytesMsg { | ||
| optional bytes bytes_type = 1; | ||
| } | ||
| message StringMsg { | ||
| optional string string_type = 1; | ||
| } | ||
|
|
||
| message Person { | ||
| optional string name = 1; | ||
| optional int32 age = 2; | ||
| } | ||
|
|
||
| message Bad { | ||
| optional bytes col_0 = 1; | ||
| optional double col_1 = 2; | ||
| optional string col_2 = 3; | ||
| optional float col_3 = 4; | ||
| optional int64 col_4 = 5; | ||
| } | ||
|
|
||
| message Actual { | ||
| optional string col_0 = 1; | ||
| optional int32 col_1 = 2; | ||
| optional float col_2 = 3; | ||
| optional bool col_3 = 4; | ||
| optional double col_4 = 5; | ||
| } | ||
|
|
||
| message oldConsumer { | ||
| optional string key = 1; | ||
| } | ||
|
|
||
| message newProducer { | ||
| optional string key = 1; | ||
| optional int32 value = 2; | ||
| } | ||
|
|
||
| message newConsumer { | ||
| optional string key = 1; | ||
| optional int32 value = 2; | ||
| optional Actual actual = 3; | ||
| } | ||
|
|
||
| message oldProducer { | ||
| optional string key = 1; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,194 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| syntax = "proto2"; | ||
|
|
||
| package org.apache.spark.sql.protobuf.protos.v2; | ||
|
|
||
| option java_outer_classname = "SimpleMessageProtos"; | ||
|
|
||
| message SimpleMessageJavaTypes { | ||
| optional int64 id = 1; | ||
| optional string string_value = 2; | ||
| optional int32 int32_value = 3; | ||
| optional int64 int64_value = 4; | ||
| optional double double_value = 5; | ||
| optional float float_value = 6; | ||
| optional bool bool_value = 7; | ||
| optional bytes bytes_value = 8; | ||
| } | ||
|
|
||
| message SimpleMessage { | ||
| optional int64 id = 1; | ||
| optional string string_value = 2; | ||
| optional int32 int32_value = 3; | ||
| optional uint32 uint32_value = 4; | ||
| optional sint32 sint32_value = 5; | ||
| optional fixed32 fixed32_value = 6; | ||
| optional sfixed32 sfixed32_value = 7; | ||
| optional int64 int64_value = 8; | ||
| optional uint64 uint64_value = 9; | ||
| optional sint64 sint64_value = 10; | ||
| optional fixed64 fixed64_value = 11; | ||
| optional sfixed64 sfixed64_value = 12; | ||
| optional double double_value = 13; | ||
| optional float float_value = 14; | ||
| optional bool bool_value = 15; | ||
| optional bytes bytes_value = 16; | ||
| } | ||
|
|
||
| message SimpleMessageRepeated { | ||
| optional string key = 1; | ||
| optional string value = 2; | ||
| enum NestedEnum { | ||
| ESTED_NOTHING = 0; | ||
| NESTED_FIRST = 1; | ||
| NESTED_SECOND = 2; | ||
| } | ||
| repeated string rstring_value = 3; | ||
| repeated int32 rint32_value = 4; | ||
| repeated bool rbool_value = 5; | ||
| repeated int64 rint64_value = 6; | ||
| repeated float rfloat_value = 7; | ||
| repeated double rdouble_value = 8; | ||
| repeated bytes rbytes_value = 9; | ||
| repeated NestedEnum rnested_enum = 10; | ||
| } | ||
|
|
||
| message BasicMessage { | ||
| optional int64 id = 1; | ||
| optional string string_value = 2; | ||
| optional int32 int32_value = 3; | ||
| optional int64 int64_value = 4; | ||
| optional double double_value = 5; | ||
| optional float float_value = 6; | ||
| optional bool bool_value = 7; | ||
| optional bytes bytes_value = 8; | ||
| } | ||
|
|
||
| message RepeatedMessage { | ||
| repeated BasicMessage basic_message = 1; | ||
| } | ||
|
|
||
| message SimpleMessageMap { | ||
| optional string key = 1; | ||
| optional string value = 2; | ||
| map<string, string> string_mapdata = 3; | ||
| map<int32, int32> int32_mapdata = 4; | ||
| map<uint32, uint32> uint32_mapdata = 5; | ||
| map<sint32, sint32> sint32_mapdata = 6; | ||
| map<fixed32, fixed32> float32_mapdata = 7; | ||
| map<sfixed32, sfixed32> sfixed32_mapdata = 8; | ||
| map<int64, int64> int64_mapdata = 9; | ||
| map<uint64, uint64> uint64_mapdata = 10; | ||
| map<sint64, sint64> sint64_mapdata = 11; | ||
| map<fixed64, fixed64> fixed64_mapdata = 12; | ||
| map<sfixed64, sfixed64> sfixed64_mapdata = 13; | ||
| map<string, double> double_mapdata = 14; | ||
| map<string, float> float_mapdata = 15; | ||
| map<bool, bool> bool_mapdata = 16; | ||
| map<string, bytes> bytes_mapdata = 17; | ||
| } | ||
|
|
||
| message BasicEnumMessage { | ||
| enum BasicEnum { | ||
| NOTHING = 0; | ||
| FIRST = 1; | ||
| SECOND = 2; | ||
| } | ||
| } | ||
|
|
||
| message SimpleMessageEnum { | ||
| optional string key = 1; | ||
| optional string value = 2; | ||
| enum NestedEnum { | ||
| NESTED_NOTHING = 0; | ||
| NESTED_FIRST = 1; | ||
| NESTED_SECOND = 2; | ||
| } | ||
| optional BasicEnumMessage.BasicEnum basic_enum = 3; | ||
| optional NestedEnum nested_enum = 4; | ||
| } | ||
|
|
||
|
|
||
| message OtherExample { | ||
| optional string other = 1; | ||
| } | ||
|
|
||
| message IncludedExample { | ||
| optional string included = 1; | ||
| optional OtherExample other = 2; | ||
| } | ||
|
|
||
| message MultipleExample { | ||
| optional IncludedExample included_example = 1; | ||
| } | ||
|
|
||
| message recursiveA { | ||
| optional string keyA = 1; | ||
| optional recursiveB messageB = 2; | ||
| } | ||
|
|
||
| message recursiveB { | ||
| optional string keyB = 1; | ||
| optional recursiveA messageA = 2; | ||
| } | ||
|
|
||
| message recursiveC { | ||
| optional string keyC = 1; | ||
| optional recursiveD messageD = 2; | ||
| } | ||
|
|
||
| message recursiveD { | ||
| optional string keyD = 1; | ||
| repeated recursiveC messageC = 2; | ||
| } | ||
|
|
||
| message requiredMsg { | ||
| optional string key = 1; | ||
| optional int32 col_1 = 2; | ||
| optional string col_2 = 3; | ||
| optional int32 col_3 = 4; | ||
| } | ||
|
|
||
| // https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/timestamp.proto | ||
| message Timestamp { | ||
| optional int64 seconds = 1; | ||
| optional int32 nanos = 2; | ||
| } | ||
|
|
||
| message timeStampMsg { | ||
| optional string key = 1; | ||
| optional Timestamp stmp = 2; | ||
| } | ||
| // https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/duration.proto | ||
| message Duration { | ||
| optional int64 seconds = 1; | ||
| optional int32 nanos = 2; | ||
| } | ||
|
|
||
| message durationMsg { | ||
| optional string key = 1; | ||
| optional Duration duration = 2; | ||
| } | ||
|
|
||
| message ProtoWithDefaults { | ||
| optional string user_name = 1; | ||
| required int32 id = 2; | ||
| optional int32 api_quota = 3 [default = 100]; // Default 100 qps. | ||
| optional string location = 4 [default = "Unknown"]; | ||
| } | ||
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| syntax = "proto2"; | ||
|
|
||
| package org.apache.spark.sql.protobuf; | ||
| option java_outer_classname = "SimpleMessageProtos"; | ||
|
|
||
| message SimpleMessage { | ||
| optional int32 age = 1; | ||
| optional string name = 2; | ||
| optional int64 score = 3; | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we still want to keep these comments ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Removed. Also added
pyspark_test.prototo this directory.