Skip to content

Commit fb8d356

Browse files
zml1206meta-codesync[bot]
authored andcommitted
feat: Support int64_t value for Spark timestampadd function (facebookincubator#15459)
Summary: Spark 4.0 change `value` from int32_t to int64_t, this PR is quickly to support it. https://github.com/apache/spark/blob/29434ea766b0fc3c3bf6eaadb43a8f931133649e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala#L3442 Pull Request resolved: facebookincubator#15459 Reviewed By: peterenescu Differential Revision: D87562354 Pulled By: kagamiori fbshipit-source-id: d748bff943af6b196225c33528b24f8c5a336fdc
1 parent 9bc0ce3 commit fb8d356

File tree

5 files changed

+32
-14
lines changed

5 files changed

+32
-14
lines changed

velox/docs/functions/spark/datetime.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ These functions support TIMESTAMP and DATE input types.
287287

288288
.. spark:function:: timestampadd(unit, value, timestamp) -> timestamp
289289
290-
Adds an interval ``value`` of type ``unit`` to ``timestamp``.
290+
Adds an int or bigint interval ``value`` of type ``unit`` to ``timestamp``.
291291
Subtraction can be performed by using a negative ``value``.
292292
Throws exception if ``unit`` is invalid.
293293
``unit`` is case insensitive and must be one of the following:

velox/functions/lib/DateTimeUtil.h

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -292,11 +292,12 @@ addToTimestamp(const Timestamp& timestamp, DateTimeUnit unit, int32_t value) {
292292
outTimestamp;
293293

294294
switch (unit) {
295-
// Year, quarter or month are not uniformly incremented in terms of number
296-
// of days. So we treat them differently.
295+
// Year, quarter, month or week are not uniformly incremented in terms of
296+
// number of days. So we treat them differently.
297297
case DateTimeUnit::kYear:
298298
case DateTimeUnit::kQuarter:
299299
case DateTimeUnit::kMonth:
300+
case DateTimeUnit::kWeek:
300301
case DateTimeUnit::kDay: {
301302
const int32_t inDate =
302303
std::chrono::duration_cast<date::days>(inTimestamp.time_since_epoch())
@@ -322,15 +323,6 @@ addToTimestamp(const Timestamp& timestamp, DateTimeUnit unit, int32_t value) {
322323
outTimestamp = inTimestamp + std::chrono::milliseconds(value);
323324
break;
324325
}
325-
case DateTimeUnit::kWeek: {
326-
const int32_t inDate =
327-
std::chrono::duration_cast<date::days>(inTimestamp.time_since_epoch())
328-
.count();
329-
const int32_t outDate = addToDate(inDate, DateTimeUnit::kDay, 7 * value);
330-
331-
outTimestamp = inTimestamp + date::days(outDate - inDate);
332-
break;
333-
}
334326
default:
335327
VELOX_UNREACHABLE("Unsupported datetime unit");
336328
}

velox/functions/sparksql/DateTimeFunctions.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,11 +1053,12 @@ template <typename T>
10531053
struct TimestampAddFunction {
10541054
VELOX_DEFINE_FUNCTION_TYPES(T);
10551055

1056+
template <typename TInput>
10561057
FOLLY_ALWAYS_INLINE void initialize(
10571058
const std::vector<TypePtr>& /*inputTypes*/,
10581059
const core::QueryConfig& config,
10591060
const arg_type<Varchar>* unitString,
1060-
const int32_t* /*value*/,
1061+
const TInput* /*value*/,
10611062
const arg_type<Timestamp>* /*timestamp*/) {
10621063
VELOX_USER_CHECK_NOT_NULL(unitString);
10631064
std::string unitStr(*unitString);
@@ -1071,10 +1072,11 @@ struct TimestampAddFunction {
10711072
sessionTimeZone_ = getTimeZoneFromConfig(config);
10721073
}
10731074

1075+
template <typename TInput>
10741076
FOLLY_ALWAYS_INLINE void call(
10751077
out_type<Timestamp>& result,
10761078
const arg_type<Varchar>& /*unitString*/,
1077-
const int32_t value,
1079+
const TInput value,
10781080
const arg_type<Timestamp>& timestamp) {
10791081
const auto unit = unit_.value();
10801082
result = addToTimestamp(unit, value, timestamp, sessionTimeZone_);

velox/functions/sparksql/registration/RegisterDatetime.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ void registerDatetimeFunctions(const std::string& prefix) {
110110
Varchar,
111111
int32_t,
112112
Timestamp>({prefix + "timestampadd"});
113+
registerFunction<
114+
TimestampAddFunction,
115+
Timestamp,
116+
Varchar,
117+
int64_t,
118+
Timestamp>({prefix + "timestampadd"});
113119
registerFunction<MonthsBetweenFunction, double, Timestamp, Timestamp, bool>(
114120
{prefix + "months_between"});
115121
}

velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1705,21 +1705,39 @@ TEST_F(DateTimeFunctionsTest, timestampadd) {
17051705
fmt::format("timestampadd('{}', c0, c1)", unit), value, timestamp);
17061706
};
17071707

1708+
const auto timestampaddBigint = [&](const std::string& unit,
1709+
std::optional<int64_t> value,
1710+
std::optional<Timestamp> timestamp) {
1711+
return evaluateOnce<Timestamp>(
1712+
fmt::format("timestampadd('{}', c0, c1)", unit), value, timestamp);
1713+
};
1714+
17081715
// Check null behaviors.
17091716
EXPECT_EQ(std::nullopt, timestampadd("second", 1, std::nullopt));
17101717
EXPECT_EQ(std::nullopt, timestampadd("month", std::nullopt, Timestamp(0, 0)));
1718+
EXPECT_EQ(
1719+
std::nullopt, timestampaddBigint("month", std::nullopt, Timestamp(0, 0)));
17111720

17121721
// Check invalid units.
17131722
VELOX_ASSERT_THROW(
17141723
timestampadd("invalid_unit", 1, Timestamp(0, 0)),
17151724
"Unsupported datetime unit: invalid_unit");
1725+
VELOX_ASSERT_THROW(
1726+
timestampaddBigint("invalid_unit", 1, Timestamp(0, 0)),
1727+
"Unsupported datetime unit: invalid_unit");
17161728

17171729
EXPECT_EQ(
17181730
Timestamp(1551348061, 999) /*2019-02-28 10:01:01.000000*/,
17191731
timestampadd(
17201732
"microsecond",
17211733
60 * 1000000 + 500,
17221734
Timestamp(1551348000, 999'500'999) /*2019-02-28 10:00:00.999500*/));
1735+
EXPECT_EQ(
1736+
Timestamp(1551348061, 999) /*2019-02-28 10:01:01.000000*/,
1737+
timestampaddBigint(
1738+
"microsecond",
1739+
60 * 1000000 + 500,
1740+
Timestamp(1551348000, 999'500'999) /*2019-02-28 10:00:00.999500*/));
17231741
EXPECT_EQ(
17241742
Timestamp(1551348061, 999'999) /*2019-02-28 10:01:01.000*/,
17251743
timestampadd(

0 commit comments

Comments
 (0)