Skip to content

[SPARK-41238][CONNECT][PYTHON] Support more built-in datatypes #38770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhengruifeng wants to merge 5 commits into from
Closed

[SPARK-41238][CONNECT][PYTHON] Support more built-in datatypes #38770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef26f8d
init
zhengruifeng Nov 23, 2022
78e7541
fix test
zhengruifeng Nov 24, 2022
4c5f279
address comments
zhengruifeng Nov 24, 2022
f5194c8
rename numeric types
zhengruifeng Nov 24, 2022
cddf350
reformat
zhengruifeng Nov 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion connector/connect/src/main/protobuf/spark/connect/expressions.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ message Expression {
bytes uuid = 28;
DataType null = 29; // a typed null literal
List list = 30;
DataType.List empty_list = 31;
DataType.Array empty_array = 31;
DataType.Map empty_map = 32;
UserDefined user_defined = 33;
}
Expand Down
123 changes: 73 additions & 50 deletions connector/connect/src/main/protobuf/spark/connect/types.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,31 +26,46 @@ option java_package = "org.apache.spark.connect.proto";
// itself but only describes it.
message DataType {
Copy link
Contributor

@grundprinzip grundprinzip Nov 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a severe breaking change, we need to make sure that at some point in time our protos become stable. In particular, here the order of the members changes.

Copy link
Contributor Author

@zhengruifeng zhengruifeng Nov 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe also need to add some UDTs(like VectorUDT) in the future, but now SQL types were completed.

oneof kind {
Boolean bool = 1;
I8 i8 = 2;
I16 i16 = 3;
I32 i32 = 5;
I64 i64 = 7;
FP32 fp32 = 10;
FP64 fp64 = 11;
String string = 12;
Binary binary = 13;
Timestamp timestamp = 14;
Date date = 16;
Time time = 17;
IntervalYear interval_year = 19;
IntervalDay interval_day = 20;
TimestampTZ timestamp_tz = 29;
UUID uuid = 32;

FixedChar fixed_char = 21;
VarChar varchar = 22;
FixedBinary fixed_binary = 23;
Decimal decimal = 24;

Struct struct = 25;
List list = 27;
Map map = 28;
NULL null = 1;

Binary binary = 2;

Boolean boolean = 3;

// Numeric types
Byte byte = 4;
Short short = 5;
Integer integer = 6;
Long long = 7;

Float float = 8;
Double double = 9;
Decimal decimal = 10;

// String types
String string = 11;
Char char = 12;
VarChar var_char = 13;

// Datatime types
Date date = 14;
Timestamp timestamp = 15;
TimestampNTZ timestamp_ntz = 16;

// Interval types
CalendarInterval calendar_interval = 17;
YearMonthInterval year_month_interval = 18;
DayTimeInterval day_time_interval = 19;

// Complex types
Array array = 20;
Struct struct = 21;
Map map = 22;


UUID uuid = 25;

FixedBinary fixed_binary = 26;

uint32 user_defined_type_reference = 31;
}
Expand All @@ -59,27 +74,27 @@ message DataType {
uint32 type_variation_reference = 1;
}

message I8 {
message Byte {
uint32 type_variation_reference = 1;
}

message I16 {
message Short {
uint32 type_variation_reference = 1;
}

message I32 {
message Integer {
uint32 type_variation_reference = 1;
}

message I64 {
message Long {
uint32 type_variation_reference = 1;
}

message FP32 {
message Float {
uint32 type_variation_reference = 1;
}

message FP64 {
message Double {
uint32 type_variation_reference = 1;
}

Expand All @@ -91,6 +106,10 @@ message DataType {
uint32 type_variation_reference = 1;
}

message NULL {
uint32 type_variation_reference = 1;
}

message Timestamp {
uint32 type_variation_reference = 1;
}
Expand All @@ -99,28 +118,32 @@ message DataType {
uint32 type_variation_reference = 1;
}

message Time {
message TimestampNTZ {
uint32 type_variation_reference = 1;
}

message TimestampTZ {
message CalendarInterval {
uint32 type_variation_reference = 1;
}

message IntervalYear {
uint32 type_variation_reference = 1;
message YearMonthInterval {
optional int32 start_field = 1;
optional int32 end_field = 2;
uint32 type_variation_reference = 3;
}

message IntervalDay {
uint32 type_variation_reference = 1;
message DayTimeInterval {
optional int32 start_field = 1;
optional int32 end_field = 2;
uint32 type_variation_reference = 3;
}

message UUID {
uint32 type_variation_reference = 1;
}

// Start compound types.
message FixedChar {
message Char {
int32 length = 1;
uint32 type_variation_reference = 2;
}
Expand All @@ -136,14 +159,14 @@ message DataType {
}

message Decimal {
int32 scale = 1;
int32 precision = 2;
optional int32 scale = 1;
optional int32 precision = 2;
uint32 type_variation_reference = 3;
}

message StructField {
DataType type = 1;
string name = 2;
string name = 1;
DataType data_type = 2;
bool nullable = 3;
map<string, string> metadata = 4;
}
Expand All @@ -153,16 +176,16 @@ message DataType {
uint32 type_variation_reference = 2;
}

message List {
DataType DataType = 1;
uint32 type_variation_reference = 2;
bool element_nullable = 3;
message Array {
DataType element_type = 1;
bool contains_null = 2;
uint32 type_variation_reference = 3;
}

message Map {
DataType key = 1;
DataType value = 2;
uint32 type_variation_reference = 3;
bool value_nullable = 4;
DataType key_type = 1;
DataType value_type = 2;
bool value_contains_null = 3;
uint32 type_variation_reference = 4;
}
}
5 changes: 3 additions & 2 deletions connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ package object dsl {
for (attr <- attrs) {
val structField = DataType.StructField.newBuilder()
structField.setName(attr.getName)
structField.setType(attr.getType)
structField.setDataType(attr.getType)
structField.setNullable(true)
structExpr.addFields(structField)
}
Expression.QualifiedAttribute
Expand All @@ -66,7 +67,7 @@ package object dsl {

/** Creates a new AttributeReference of type int */
def int: Expression.QualifiedAttribute = protoQualifiedAttrWithType(
DataType.newBuilder().setI32(DataType.I32.newBuilder()).build())
DataType.newBuilder().setInteger(DataType.Integer.newBuilder()).build())

private def protoQualifiedAttrWithType(dataType: DataType): Expression.QualifiedAttribute =
Expression.QualifiedAttribute
Expand Down
Loading