diff --git a/pyvelox/pyvelox.h b/pyvelox/pyvelox.h index 48c40f90622..41c58167055 100644 --- a/pyvelox/pyvelox.h +++ b/pyvelox/pyvelox.h @@ -398,8 +398,7 @@ static void addVectorBindings( TypeKind::REAL, TypeKind::DOUBLE, TypeKind::VARBINARY, - TypeKind::TIMESTAMP, - TypeKind::DATE}; + TypeKind::TIMESTAMP}; for (int i = 0; i < sizeof(supportedTypes) / sizeof(supportedTypes[0]); i++) { VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( diff --git a/velox/common/memory/ByteStream.h b/velox/common/memory/ByteStream.h index 522bcc0cb64..1854cbe5456 100644 --- a/velox/common/memory/ByteStream.h +++ b/velox/common/memory/ByteStream.h @@ -393,13 +393,6 @@ inline int128_t ByteStream::read() { return value; } -template <> -inline Date ByteStream::read() { - Date value; - readBytes(reinterpret_cast(&value), sizeof(value)); - return value; -} - class IOBufOutputStream : public OutputStream { public: explicit IOBufOutputStream( diff --git a/velox/connectors/hive/HivePartitionFunction.cpp b/velox/connectors/hive/HivePartitionFunction.cpp index 624b413aaea..f9e7e1e17a3 100644 --- a/velox/connectors/hive/HivePartitionFunction.cpp +++ b/velox/connectors/hive/HivePartitionFunction.cpp @@ -173,16 +173,6 @@ void hashTyped( abstractHashTyped(values, size, mix, hashTimestamp, hashes); } -template <> -void hashTyped( - const DecodedVector& values, - vector_size_t size, - bool mix, - std::vector& hashes) { - auto hashDate = [](const Date& value) { return value.days(); }; - abstractHashTyped(values, size, mix, hashDate, hashes); -} - void hash( const DecodedVector& values, TypeKind typeKind, diff --git a/velox/connectors/hive/HivePartitionUtil.cpp b/velox/connectors/hive/HivePartitionUtil.cpp index e71c24f333b..3adf9d38a82 100644 --- a/velox/connectors/hive/HivePartitionUtil.cpp +++ b/velox/connectors/hive/HivePartitionUtil.cpp @@ -28,10 +28,8 @@ namespace facebook::velox::connector::hive { case TypeKind::BIGINT: \ case TypeKind::VARCHAR: \ case TypeKind::VARBINARY: \ - case TypeKind::DATE: { \ return VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( \ TEMPLATE_FUNC, typeKind, __VA_ARGS__); \ - } \ default: \ VELOX_UNSUPPORTED( \ "Unsupported partition type: {}", mapTypeKindToName(typeKind)); \ @@ -49,11 +47,6 @@ inline std::string makePartitionValueString(bool value) { return value ? "true" : "false"; } -template <> -inline std::string makePartitionValueString(Date value) { - return value.toString(); -} - template std::pair makePartitionKeyValueString( const BaseVector* partitionVector, @@ -73,12 +66,21 @@ std::vector> extractPartitionKeyValues( vector_size_t row) { std::vector> partitionKeyValues; for (auto i = 0; i < partitionsVector->childrenSize(); i++) { - partitionKeyValues.push_back(PARTITION_TYPE_DISPATCH( - makePartitionKeyValueString, - partitionsVector->childAt(i)->typeKind(), - partitionsVector->childAt(i)->loadedVector(), - row, - asRowType(partitionsVector->type())->nameOf(i))); + if (partitionsVector->childAt(i)->type()->isDate()) { + auto partitionVector = partitionsVector->childAt(i)->loadedVector(); + auto partitionName = asRowType(partitionsVector->type())->nameOf(i); + partitionKeyValues.push_back( + {partitionName, + DATE()->toString( + partitionVector->as>()->valueAt(row))}); + } else { + partitionKeyValues.push_back(PARTITION_TYPE_DISPATCH( + makePartitionKeyValueString, + partitionsVector->childAt(i)->typeKind(), + partitionsVector->childAt(i)->loadedVector(), + row, + asRowType(partitionsVector->type())->nameOf(i))); + } } return partitionKeyValues; } diff --git a/velox/connectors/hive/benchmarks/HivePartitionFunctionBenchmark.cpp b/velox/connectors/hive/benchmarks/HivePartitionFunctionBenchmark.cpp index 90cb088ea7b..36532b495f5 100644 --- a/velox/connectors/hive/benchmarks/HivePartitionFunctionBenchmark.cpp +++ b/velox/connectors/hive/benchmarks/HivePartitionFunctionBenchmark.cpp @@ -39,8 +39,7 @@ constexpr std::array kSupportedTypes{ TypeKind::REAL, TypeKind::DOUBLE, TypeKind::VARCHAR, - TypeKind::TIMESTAMP, - TypeKind::DATE}; + TypeKind::TIMESTAMP}; class HivePartitionFunctionBenchmark : public functions::test::FunctionBenchmarkBase { @@ -265,23 +264,6 @@ BENCHMARK_RELATIVE(timestampManyRowsManyBuckets) { } BENCHMARK_DRAW_LINE(); - -BENCHMARK(dateFewRowsFewBuckets) { - benchmarkFew->runFew(); -} - -BENCHMARK_RELATIVE(dateFewRowsManyBuckets) { - benchmarkFew->runMany(); -} - -BENCHMARK(dateManyRowsFewBuckets) { - benchmarkMany->runFew(); -} - -BENCHMARK_RELATIVE(dateManyRowsManyBuckets) { - benchmarkMany->runMany(); -} - } // namespace int main(int argc, char** argv) { diff --git a/velox/connectors/hive/tests/HivePartitionFunctionTest.cpp b/velox/connectors/hive/tests/HivePartitionFunctionTest.cpp index 8817fcc4cc4..4f763d80f16 100644 --- a/velox/connectors/hive/tests/HivePartitionFunctionTest.cpp +++ b/velox/connectors/hive/tests/HivePartitionFunctionTest.cpp @@ -258,11 +258,12 @@ TEST_F(HivePartitionFunctionTest, timestamp) { } TEST_F(HivePartitionFunctionTest, date) { - auto values = makeNullableFlatVector( + auto values = makeNullableFlatVector( {std::nullopt, - Date(2'000'000'000), - Date(std::numeric_limits::min()), - Date(std::numeric_limits::max())}); + 2'000'000'000, + std::numeric_limits::min(), + std::numeric_limits::max()}, + DATE()); assertPartitions(values, 1, {0, 0, 0, 0}); assertPartitions(values, 2, {0, 0, 0, 1}); diff --git a/velox/connectors/hive/tests/HivePartitionUtilTest.cpp b/velox/connectors/hive/tests/HivePartitionUtilTest.cpp index ea49cf51420..7bd16286dec 100644 --- a/velox/connectors/hive/tests/HivePartitionUtilTest.cpp +++ b/velox/connectors/hive/tests/HivePartitionUtilTest.cpp @@ -75,7 +75,7 @@ TEST_F(HivePartitionUtilTest, partitionName) { makeFlatVector(std::vector{1000}), makeFlatVector(std::vector{10000}), makeDictionary(std::vector{"str1000"}), - makeConstant(Date(10000), 1)}); + makeConstant(10000, 1, DATE())}); std::vector expectedPartitionKeyValues{ "flat_bool_col=false", diff --git a/velox/connectors/hive/tests/PartitionIdGeneratorTest.cpp b/velox/connectors/hive/tests/PartitionIdGeneratorTest.cpp index 98e665078ce..b37871b259c 100644 --- a/velox/connectors/hive/tests/PartitionIdGeneratorTest.cpp +++ b/velox/connectors/hive/tests/PartitionIdGeneratorTest.cpp @@ -33,7 +33,7 @@ TEST_F(PartitionIdGeneratorTest, consecutiveIdsSingleKey) { auto input = makeRowVector( {makeFlatVector(numPartitions * 3, [&](auto row) { return StringView::makeInline( - Date(18000 + row % numPartitions).toString()); + DATE()->toString(18000 + row % numPartitions)); })}); raw_vector ids; @@ -56,7 +56,7 @@ TEST_F(PartitionIdGeneratorTest, consecutiveIdsMultipleKeys) { makeFlatVector( 1'000, [&](auto row) { - return StringView::makeInline(Date(18000 + row % 5).toString()); + return StringView::makeInline(DATE()->toString(18000 + row % 5)); }), makeFlatVector(1'000, [&](auto row) { return row % 17; }), }); @@ -111,7 +111,7 @@ TEST_F(PartitionIdGeneratorTest, stableIdsMultipleKeys) { makeFlatVector( size, [](auto row) { - return StringView::makeInline(Date(18000 + row % 3).toString()); + return StringView::makeInline(DATE()->toString(18000 + row % 3)); }), makeFlatVector(size, [](auto row) { return row % 7; }), }); @@ -124,7 +124,7 @@ TEST_F(PartitionIdGeneratorTest, stableIdsMultipleKeys) { makeFlatVector( size, [](auto row) { - return StringView::makeInline(Date(18000 + row % 5).toString()); + return StringView::makeInline(DATE()->toString(18000 + row % 5)); }), makeFlatVector(size, [](auto row) { return row % 17; }), }); diff --git a/velox/connectors/tpch/tests/TpchConnectorTest.cpp b/velox/connectors/tpch/tests/TpchConnectorTest.cpp index ab2a22199c3..7685bebf516 100644 --- a/velox/connectors/tpch/tests/TpchConnectorTest.cpp +++ b/velox/connectors/tpch/tests/TpchConnectorTest.cpp @@ -277,8 +277,8 @@ TEST_F(TpchConnectorTest, orderDateCount) { .planNode(); auto output = getResults(plan, {makeTpchSplit()}); - auto orderDate = output->childAt(0)->asFlatVector(); - EXPECT_EQ("1992-01-01", orderDate->valueAt(0).toString()); + auto orderDate = output->childAt(0)->asFlatVector(); + EXPECT_EQ("1992-01-01", DATE()->toString(orderDate->valueAt(0))); // Match with count obtained from Java. EXPECT_EQ(9, orderDate->size()); } diff --git a/velox/docs/develop/types.rst b/velox/docs/develop/types.rst index 12aa0a98212..5613ba73978 100644 --- a/velox/docs/develop/types.rst +++ b/velox/docs/develop/types.rst @@ -23,7 +23,6 @@ SMALLINT int16_t 2 INTEGER int32_t 4 BIGINT int64_t 8 HUGEINT int128_t 16 -DATE struct Date 8 REAL float 4 DOUBLE double 8 TIMESTAMP struct Timestamp 16 @@ -84,14 +83,12 @@ their corresponding physical type. ====================== ====================================================== Logical Type Physical Type ====================== ====================================================== +DATE INTEGER DECIMAL BIGINT if precision <= 18, HUGEINT if precision >= 19 INTERVAL DAY TO SECOND BIGINT INTERVAL YEAR TO MONTH INTEGER ====================== ====================================================== -We are in the process of migrating (:pr:`4744`) DATE type to a logical type backed -by BIGINT. - DECIMAL type carries additional `precision`, and `scale` information. `Precision` is the number of digits in a number. `Scale` is the number of digits to the right of the decimal diff --git a/velox/duckdb/conversion/DuckConversion.cpp b/velox/duckdb/conversion/DuckConversion.cpp index c95dafd4bf2..7764b3b99b0 100644 --- a/velox/duckdb/conversion/DuckConversion.cpp +++ b/velox/duckdb/conversion/DuckConversion.cpp @@ -59,6 +59,7 @@ LogicalType fromVeloxType(const TypePtr& type) { auto [precision, scale] = getDecimalPrecisionScale(*type); return LogicalType::DECIMAL(precision, scale); } + switch (type->kind()) { case TypeKind::BOOLEAN: return LogicalType::BOOLEAN; @@ -70,6 +71,9 @@ LogicalType fromVeloxType(const TypePtr& type) { if (type->isIntervalYearMonth()) { return LogicalType::INTERVAL; } + if (type->isDate()) { + return LogicalType::DATE; + } return LogicalType::INTEGER; case TypeKind::BIGINT: if (type->isIntervalDayTime()) { @@ -84,8 +88,6 @@ LogicalType fromVeloxType(const TypePtr& type) { return LogicalType::VARCHAR; case TypeKind::TIMESTAMP: return LogicalType::TIMESTAMP; - case TypeKind::DATE: - return LogicalType::DATE; case TypeKind::ARRAY: return LogicalType::LIST(fromVeloxType(type->childAt(0))); case TypeKind::MAP: @@ -207,7 +209,7 @@ variant duckValueToVariant(const Value& val) { case LogicalTypeId::BLOB: return variant::binary(val.GetValue()); case LogicalTypeId::DATE: - return variant::date(val.GetValue<::duckdb::date_t>().days); + return variant(val.GetValue<::duckdb::date_t>().days); default: throw std::runtime_error( "unsupported type for duckdb value -> velox variant conversion: " + diff --git a/velox/duckdb/conversion/DuckConversion.h b/velox/duckdb/conversion/DuckConversion.h index 0f1fc4e5faa..de5d2fc4fdf 100644 --- a/velox/duckdb/conversion/DuckConversion.h +++ b/velox/duckdb/conversion/DuckConversion.h @@ -198,15 +198,15 @@ struct DuckTimestampConversion { struct DuckDateConversion { typedef ::duckdb::date_t DUCK_TYPE; - typedef Date VELOX_TYPE; + typedef int32_t VELOX_TYPE; static ::duckdb::date_t toDuck( - const Date& input, + const int32_t& input, ::duckdb::Vector& /* unused */) { - return ::duckdb::Date::EpochDaysToDate(input.days()); + return ::duckdb::Date::EpochDaysToDate(input); } - static Date toVelox(const ::duckdb::date_t& input) { - return Date(::duckdb::Date::EpochDays(input)); + static int32_t toVelox(const ::duckdb::date_t& input) { + return ::duckdb::Date::EpochDays(input); } }; diff --git a/velox/duckdb/conversion/tests/DuckWrapperTest.cpp b/velox/duckdb/conversion/tests/DuckWrapperTest.cpp index 6c8e28e8697..52b00bbb9e8 100644 --- a/velox/duckdb/conversion/tests/DuckWrapperTest.cpp +++ b/velox/duckdb/conversion/tests/DuckWrapperTest.cpp @@ -136,7 +136,7 @@ TEST_F(BaseDuckWrapperTest, scalarTypes) { verifyUnaryResult("SELECT 1::DOUBLE", {1.0}); // date/timestamp - verifyUnaryResult("SELECT DATE '1992-01-01'", {Date(8035)}); + verifyUnaryResult("SELECT DATE '1992-01-01'", {8035}); verifyUnaryResult( "SELECT TIMESTAMP '1992-01-01 13:04:20'", {Timestamp(694271060, 0)}); @@ -189,9 +189,9 @@ TEST_F(BaseDuckWrapperTest, types) { {false, false, false, true}); // date/timestamp - verifyUnaryResult( + verifyUnaryResult( "SELECT i FROM (VALUES (DATE '1992-01-01'), (NULL)) tbl(i)", - {Date(8035), Date(0)}, + {8035, 0}, {false, true}); verifyUnaryResult( "SELECT i FROM (VALUES (TIMESTAMP '1992-01-01 13:04:20'), (NULL)) tbl(i)", @@ -220,8 +220,7 @@ TEST_F(BaseDuckWrapperTest, tpchSF1) { execute("CALL dbgen(sf=0.01)"); // test conversion of date, decimal and string verifyUnaryResult("SELECT l_discount FROM lineitem LIMIT 1", {4}); - verifyUnaryResult( - "SELECT l_shipdate FROM lineitem LIMIT 1", {Date(9568)}); + verifyUnaryResult("SELECT l_shipdate FROM lineitem LIMIT 1", {9568}); verifyUnaryResult( "SELECT l_comment FROM lineitem LIMIT 1", {StringView("egular courts above the")}); diff --git a/velox/dwio/common/SelectiveColumnReader.cpp b/velox/dwio/common/SelectiveColumnReader.cpp index 2a49d60acad..5af2476ed99 100644 --- a/velox/dwio/common/SelectiveColumnReader.cpp +++ b/velox/dwio/common/SelectiveColumnReader.cpp @@ -179,9 +179,6 @@ void SelectiveColumnReader::getIntValues( VELOX_FAIL("Unsupported value size: {}", valueSize_); } break; - case TypeKind::DATE: - getFlatValues(rows, result, requestedType); - break; case TypeKind::HUGEINT: getFlatValues(rows, result, requestedType); break; diff --git a/velox/dwio/common/tests/utils/BatchMaker.cpp b/velox/dwio/common/tests/utils/BatchMaker.cpp index bb8f2d0cee2..52ce6973b80 100644 --- a/velox/dwio/common/tests/utils/BatchMaker.cpp +++ b/velox/dwio/common/tests/utils/BatchMaker.cpp @@ -279,21 +279,6 @@ VectorPtr BatchMaker::createVector( isNullAt); } -template <> -VectorPtr BatchMaker::createVector( - const std::shared_ptr& /* unused */, - size_t size, - MemoryPool& pool, - std::mt19937& gen, - std::function isNullAt) { - return createScalar( - size, - gen, - [&gen]() { return Date(Random::rand32(gen)); }, - pool, - isNullAt); -} - template <> VectorPtr BatchMaker::createVector( const std::shared_ptr& type, diff --git a/velox/dwio/common/tests/utils/FilterGenerator.cpp b/velox/dwio/common/tests/utils/FilterGenerator.cpp index 01122ace4ec..9adc3ca0695 100644 --- a/velox/dwio/common/tests/utils/FilterGenerator.cpp +++ b/velox/dwio/common/tests/utils/FilterGenerator.cpp @@ -84,11 +84,6 @@ VectorPtr getChildBySubfield( uint32_t AbstractColumnStats::counter_ = 0; -template <> -int64_t ColumnStats::getIntegerValue(const Date& value) { - return value.days(); -} - template <> std::unique_ptr ColumnStats::makeRangeFilter( const FilterSpec& filterSpec) { @@ -413,9 +408,6 @@ SubfieldFilters FilterGenerator::makeSubfieldFilters( case TypeKind::BIGINT: stats = makeStats(vector->type(), rowType_); break; - case TypeKind::DATE: - stats = makeStats(vector->type(), rowType_); - break; case TypeKind::VARCHAR: stats = makeStats(vector->type(), rowType_); break; diff --git a/velox/dwio/dwrf/common/FileMetadata.cpp b/velox/dwio/dwrf/common/FileMetadata.cpp index 482f6d98747..3d6629a13c9 100644 --- a/velox/dwio/dwrf/common/FileMetadata.cpp +++ b/velox/dwio/dwrf/common/FileMetadata.cpp @@ -90,8 +90,9 @@ TypeKind TypeWrapper::kind() const { return TypeKind::ROW; case proto::orc::Type_Kind_VARCHAR: return TypeKind::VARCHAR; + // Date is a logical type of INTEGER (for the number of days since EPOCH). case proto::orc::Type_Kind_DATE: - return TypeKind::DATE; + return TypeKind::INTEGER; case proto::orc::Type_Kind_DECIMAL: case proto::orc::Type_Kind_CHAR: case proto::orc::Type_Kind_TIMESTAMP_INSTANT: diff --git a/velox/dwio/dwrf/reader/ColumnReader.cpp b/velox/dwio/dwrf/reader/ColumnReader.cpp index d58dd7ad9ee..e1c54c0ffaf 100644 --- a/velox/dwio/dwrf/reader/ColumnReader.cpp +++ b/velox/dwio/dwrf/reader/ColumnReader.cpp @@ -2380,14 +2380,6 @@ std::unique_ptr ColumnReader::build( case TypeKind::TIMESTAMP: return std::make_unique( dataType, stripe, streamLabels, std::move(flatMapContext)); - case TypeKind::DATE: - return std::make_unique>( - dataType, - requestedType->type, - stripe, - streamLabels, - dwio::common::INT_BYTE_SIZE, - std::move(flatMapContext)); default: DWIO_RAISE("buildReader unhandled type"); } diff --git a/velox/dwio/dwrf/reader/DwrfReader.cpp b/velox/dwio/dwrf/reader/DwrfReader.cpp index 98f8d4322ad..e60aa84d0ae 100644 --- a/velox/dwio/dwrf/reader/DwrfReader.cpp +++ b/velox/dwio/dwrf/reader/DwrfReader.cpp @@ -480,9 +480,6 @@ std::optional DwrfRowReader::estimatedRowSizeHelper( case TypeKind::DOUBLE: { return valueCount * sizeof(double); } - case TypeKind::DATE: { - return valueCount * sizeof(uint32_t); - } case TypeKind::VARCHAR: { auto stringStats = dynamic_cast(&s); @@ -626,7 +623,6 @@ uint64_t maxStreamsForType(const TypeWrapper& type) { case TypeKind::REAL: case TypeKind::DOUBLE: case TypeKind::BOOLEAN: - case TypeKind::DATE: case TypeKind::ARRAY: case TypeKind::MAP: return 2; diff --git a/velox/dwio/dwrf/reader/ReaderBase.cpp b/velox/dwio/dwrf/reader/ReaderBase.cpp index 357fbf65a37..4d44d9a8f84 100644 --- a/velox/dwio/dwrf/reader/ReaderBase.cpp +++ b/velox/dwio/dwrf/reader/ReaderBase.cpp @@ -294,7 +294,6 @@ std::shared_ptr ReaderBase::convertType( case TypeKind::VARCHAR: case TypeKind::VARBINARY: case TypeKind::TIMESTAMP: - case TypeKind::DATE: return createScalarType(type.kind()); case TypeKind::ARRAY: return ARRAY(convertType( diff --git a/velox/dwio/dwrf/test/ReaderTest.cpp b/velox/dwio/dwrf/test/ReaderTest.cpp index ec4d8567572..52cd1ef2973 100644 --- a/velox/dwio/dwrf/test/ReaderTest.cpp +++ b/velox/dwio/dwrf/test/ReaderTest.cpp @@ -1739,11 +1739,11 @@ TEST(TestReader, testOrcReaderDate) { int year = 1900; while (rowReader->next(1000, batch)) { auto rowVector = batch->as(); - auto dates = rowVector->childAt(1)->as>(); + auto dates = rowVector->childAt(1)->as>(); std::stringstream stream; stream << year << "-12-25"; - EXPECT_EQ(stream.str(), dates->valueAt(0).toString()); + EXPECT_EQ(stream.str(), DATE()->toString(dates->valueAt(0))); for (size_t i = 1; i < rowVector->size(); ++i) { EXPECT_EQ(dates->valueAt(0), dates->valueAt(i)); diff --git a/velox/dwio/dwrf/utils/ProtoUtils.cpp b/velox/dwio/dwrf/utils/ProtoUtils.cpp index 97906e7e8ec..405d2e79ddf 100644 --- a/velox/dwio/dwrf/utils/ProtoUtils.cpp +++ b/velox/dwio/dwrf/utils/ProtoUtils.cpp @@ -41,7 +41,6 @@ CREATE_TYPE_TRAIT(DOUBLE, DOUBLE) CREATE_TYPE_TRAIT(VARCHAR, STRING) CREATE_TYPE_TRAIT(VARBINARY, BINARY) CREATE_TYPE_TRAIT(TIMESTAMP, TIMESTAMP) -CREATE_TYPE_TRAIT(DATE, LONG) CREATE_TYPE_TRAIT(ARRAY, LIST) CREATE_TYPE_TRAIT(MAP, MAP) CREATE_TYPE_TRAIT(ROW, STRUCT) diff --git a/velox/dwio/dwrf/writer/WriterContext.h b/velox/dwio/dwrf/writer/WriterContext.h index 10196a4b583..4a6206545df 100644 --- a/velox/dwio/dwrf/writer/WriterContext.h +++ b/velox/dwio/dwrf/writer/WriterContext.h @@ -438,7 +438,6 @@ class WriterContext : public CompressionBufferPool { case TypeKind::VARCHAR: case TypeKind::VARBINARY: case TypeKind::TIMESTAMP: - case TypeKind::DATE: physicalSizeAggregators_.emplace( type.id, std::make_unique(parent)); break; diff --git a/velox/dwio/parquet/reader/ParquetColumnReader.cpp b/velox/dwio/parquet/reader/ParquetColumnReader.cpp index e670ef14ef0..bdec3011df5 100644 --- a/velox/dwio/parquet/reader/ParquetColumnReader.cpp +++ b/velox/dwio/parquet/reader/ParquetColumnReader.cpp @@ -44,7 +44,6 @@ std::unique_ptr ParquetColumnReader::build( case TypeKind::BIGINT: case TypeKind::SMALLINT: case TypeKind::TINYINT: - case TypeKind::DATE: case TypeKind::HUGEINT: return std::make_unique( dataType, dataType, params, scanSpec); diff --git a/velox/dwio/parquet/reader/ParquetColumnReader.h b/velox/dwio/parquet/reader/ParquetColumnReader.h index 4257490a1bb..5b4454819ba 100644 --- a/velox/dwio/parquet/reader/ParquetColumnReader.h +++ b/velox/dwio/parquet/reader/ParquetColumnReader.h @@ -26,7 +26,6 @@ inline int32_t parquetSizeOfIntKind(TypeKind kind) { case TypeKind::TINYINT: case TypeKind::SMALLINT: case TypeKind::INTEGER: - case TypeKind::DATE: return 4; case TypeKind::BIGINT: return 8; diff --git a/velox/dwio/parquet/tests/duckdb_reader/ParquetReaderTest.cpp b/velox/dwio/parquet/tests/duckdb_reader/ParquetReaderTest.cpp index 549e971dc17..76a5c847db9 100644 --- a/velox/dwio/parquet/tests/duckdb_reader/ParquetReaderTest.cpp +++ b/velox/dwio/parquet/tests/duckdb_reader/ParquetReaderTest.cpp @@ -59,12 +59,6 @@ class ParquetReaderTest : public ParquetReaderTestBase { } }; -template <> -VectorPtr ParquetReaderTestBase::rangeVector(size_t size, Date start) { - return vectorMaker_->flatVector( - size, [&](auto row) { return Date(start.days() + row); }); -} - TEST_F(ParquetReaderTest, readSampleFull) { // sample.parquet holds two columns (a: BIGINT, b: DOUBLE) and // 20 rows (10 rows per group). Group offsets are 153 and 614. @@ -199,14 +193,14 @@ TEST_F(ParquetReaderTest, dateRead) { auto type = reader->typeWithId(); EXPECT_EQ(type->size(), 1ULL); auto col0 = type->childAt(0); - EXPECT_EQ(col0->type->kind(), TypeKind::DATE); + EXPECT_EQ(col0->type, DATE()); auto rowReaderOpts = getReaderOpts(dateSchema()); auto scanSpec = makeScanSpec(dateSchema()); rowReaderOpts.setScanSpec(scanSpec); auto rowReader = reader->createRowReader(rowReaderOpts); - auto expected = vectorMaker_->rowVector({rangeVector(25, -5)}); + auto expected = vectorMaker_->rowVector({rangeVector(25, -5)}); assertReadExpected(dateSchema(), *rowReader, expected, *pool_); } @@ -215,7 +209,7 @@ TEST_F(ParquetReaderTest, dateFilter) { FilterMap filters; filters.insert({"date", exec::between(5, 14)}); - auto expected = vectorMaker_->rowVector({rangeVector(10, 5)}); + auto expected = vectorMaker_->rowVector({rangeVector(10, 5)}); assertReadWithFilters( "date.parquet", dateSchema(), std::move(filters), expected); diff --git a/velox/dwio/parquet/tests/reader/E2EFilterTest.cpp b/velox/dwio/parquet/tests/reader/E2EFilterTest.cpp index b2fe8da33ba..162bd72d448 100644 --- a/velox/dwio/parquet/tests/reader/E2EFilterTest.cpp +++ b/velox/dwio/parquet/tests/reader/E2EFilterTest.cpp @@ -554,7 +554,7 @@ TEST_F(E2EFilterTest, date) { testWithTypes( "date_val:date", [&]() { - makeIntDistribution( + makeIntDistribution( "date_val", 10, // min 100, // max diff --git a/velox/dwio/parquet/tests/reader/ParquetReaderTest.cpp b/velox/dwio/parquet/tests/reader/ParquetReaderTest.cpp index 69bd4613e81..2141546b37d 100644 --- a/velox/dwio/parquet/tests/reader/ParquetReaderTest.cpp +++ b/velox/dwio/parquet/tests/reader/ParquetReaderTest.cpp @@ -170,7 +170,7 @@ TEST_F(ParquetReaderTest, parseDate) { auto type = reader.typeWithId(); EXPECT_EQ(type->size(), 1ULL); auto col0 = type->childAt(0); - EXPECT_EQ(col0->type->kind(), TypeKind::DATE); + EXPECT_EQ(col0->type, DATE()); EXPECT_EQ(type->childByName("date"), col0); } diff --git a/velox/dwio/type/fbhive/HiveTypeParser.cpp b/velox/dwio/type/fbhive/HiveTypeParser.cpp index cda21a43066..372035f4dbe 100644 --- a/velox/dwio/type/fbhive/HiveTypeParser.cpp +++ b/velox/dwio/type/fbhive/HiveTypeParser.cpp @@ -50,7 +50,7 @@ HiveTypeParser::HiveTypeParser() { setupMetadata("smallint"); setupMetadata({"integer", "int"}); setupMetadata("bigint"); - setupMetadata("date"); + setupMetadata("date"); setupMetadata({"float", "real"}); setupMetadata("double"); setupMetadata("decimal"); diff --git a/velox/dwio/type/fbhive/tests/HiveTypeParserTests.cpp b/velox/dwio/type/fbhive/tests/HiveTypeParserTests.cpp index 08b78700b32..9866c65c01d 100644 --- a/velox/dwio/type/fbhive/tests/HiveTypeParserTests.cpp +++ b/velox/dwio/type/fbhive/tests/HiveTypeParserTests.cpp @@ -41,13 +41,12 @@ TEST(FbHive, typeParserPrimitive) { validate("smallint"); validate("int"); validate("bigint"); - validate("date"); validate("float"); validate("double"); validate("string"); validate("varchar"); validate("varchar(16)"); - + validate("date"); validate(" int "); } diff --git a/velox/exec/VectorHasher.cpp b/velox/exec/VectorHasher.cpp index 774ee4917d7..d2534a7eea7 100644 --- a/velox/exec/VectorHasher.cpp +++ b/velox/exec/VectorHasher.cpp @@ -40,9 +40,6 @@ namespace facebook::velox::exec { case TypeKind::BIGINT: { \ return TEMPLATE_FUNC(__VA_ARGS__); \ } \ - case TypeKind::DATE: { \ - return TEMPLATE_FUNC(__VA_ARGS__); \ - } \ case TypeKind::VARCHAR: \ case TypeKind::VARBINARY: { \ return TEMPLATE_FUNC(__VA_ARGS__); \ @@ -678,7 +675,6 @@ void extendRange( extendRange(reserve, min, max); break; case TypeKind::INTEGER: - case TypeKind::DATE: extendRange(reserve, min, max); break; case TypeKind::BIGINT: diff --git a/velox/exec/VectorHasher.h b/velox/exec/VectorHasher.h index 87c572eeb5d..8c450e56470 100644 --- a/velox/exec/VectorHasher.h +++ b/velox/exec/VectorHasher.h @@ -45,13 +45,6 @@ class UniqueValue { } } - explicit UniqueValue(Date value) { - // The number of valid bytes of Date stored in data_ is - // (int64_t)value.days(). - size_ = sizeof(int64_t); - data_ = value.days(); - } - uint32_t size() const { return size_; } @@ -301,7 +294,6 @@ class VectorHasher { case TypeKind::BIGINT: case TypeKind::VARCHAR: case TypeKind::VARBINARY: - case TypeKind::DATE: return true; default: return false; @@ -577,11 +569,6 @@ class VectorHasher { uint64_t distinctStringsBytes_ = 0; }; -template <> -inline int64_t VectorHasher::toInt64(Date value) const { - return value.days(); -} - template <> bool VectorHasher::makeValueIdsForRows( char** groups, diff --git a/velox/exec/tests/VectorHasherTest.cpp b/velox/exec/tests/VectorHasherTest.cpp index 2a07b604093..f22d99ee87a 100644 --- a/velox/exec/tests/VectorHasherTest.cpp +++ b/velox/exec/tests/VectorHasherTest.cpp @@ -450,11 +450,11 @@ TEST_F(VectorHasherTest, integerIds) { TEST_F(VectorHasherTest, dateIds) { auto vector = BaseVector::create(DATE(), 100, pool_.get()); - auto* dates = vector->as>(); + auto* dates = vector->as>(); static constexpr int32_t kMin = std::numeric_limits::min(); dates->setNull(0, true); for (auto i = 0; i < 99; ++i) { - dates->set(i + 1, Date(kMin + i * 10)); + dates->set(i + 1, kMin + i * 10); } auto hasher = exec::VectorHasher::create(DATE(), 1); raw_vector hashes(dates->size()); @@ -492,7 +492,7 @@ TEST_F(VectorHasherTest, dateIds) { for (auto count = 0; count < 1000; ++count) { vector_size_t index = 0; for (int64_t value = count * 100; value < count * 100 + 100; ++value) { - dates->set(index++, Date(value)); + dates->set(index++, value); } hasher->decode(*vector, rows); hasher->computeValueIds(rows, hashes); diff --git a/velox/exec/tests/utils/QueryAssertions.cpp b/velox/exec/tests/utils/QueryAssertions.cpp index e86be574125..22a905b272f 100644 --- a/velox/exec/tests/utils/QueryAssertions.cpp +++ b/velox/exec/tests/utils/QueryAssertions.cpp @@ -57,15 +57,6 @@ ::duckdb::Value duckValueAt( veloxTimestampToDuckDB(vector->as>()->valueAt(index))); } -template <> -::duckdb::Value duckValueAt( - const VectorPtr& vector, - vector_size_t index) { - using T = typename KindToFlatVector::WrapperType; - return ::duckdb::Value::DATE(::duckdb::Date::EpochDaysToDate( - vector->as>()->valueAt(index).days())); -} - template <> ::duckdb::Value duckValueAt( const VectorPtr& vector, @@ -220,19 +211,12 @@ velox::variant variantAt( dataChunk->GetValue(column, row).GetValue<::duckdb::timestamp_t>())); } -template <> -velox::variant variantAt( - ::duckdb::DataChunk* dataChunk, - int32_t row, - int32_t column) { - return velox::variant::date(::duckdb::Date::EpochDays( - dataChunk->GetValue(column, row).GetValue<::duckdb::date_t>())); -} - template velox::variant variantAt(const ::duckdb::Value& value) { if (value.type() == ::duckdb::LogicalType::INTERVAL) { return ::duckdb::Interval::GetMicro(value.GetValue<::duckdb::interval_t>()); + } else if (value.type() == ::duckdb::LogicalType::DATE) { + return ::duckdb::Date::EpochDays(value.GetValue<::duckdb::date_t>()); } else { // NOTE: duckdb only support native cpp type for GetValue so we need to use // DeepCopiedType instead of WrapperType here. @@ -247,12 +231,6 @@ velox::variant variantAt(const ::duckdb::Value& value) { duckdbTimestampToVelox(value.GetValue<::duckdb::timestamp_t>())); } -template <> -velox::variant variantAt(const ::duckdb::Value& value) { - return velox::variant::date( - ::duckdb::Date::EpochDays(value.GetValue<::duckdb::date_t>())); -} - variant nullVariant(const TypePtr& type) { return variant(type->kind()); } @@ -375,6 +353,10 @@ std::vector materialize( auto value = variant(::duckdb::Interval::GetMicro( dataChunk->GetValue(j, i).GetValue<::duckdb::interval_t>())); row.push_back(value); + } else if (type->isDate()) { + auto value = variant(::duckdb::Date::EpochDays( + dataChunk->GetValue(j, i).GetValue<::duckdb::date_t>())); + row.push_back(value); } else { auto value = VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( variantAt, typeKind, dataChunk, i, j); @@ -827,6 +809,10 @@ void DuckDbQueryRunner::createTable( auto value = ::duckdb::Value::INTERVAL( 0, 0, columnVector->as>()->valueAt(row)); appender.Append(value); + } else if (type->isDate()) { + auto value = ::duckdb::Value::DATE(::duckdb::Date::EpochDaysToDate( + columnVector->as>()->valueAt(row))); + appender.Append(value); } else { auto value = VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( duckValueAt, type->kind(), columnVector, row); diff --git a/velox/exec/tests/utils/TpchQueryBuilder.cpp b/velox/exec/tests/utils/TpchQueryBuilder.cpp index 3c06c20d3aa..c1c73c9cf23 100644 --- a/velox/exec/tests/utils/TpchQueryBuilder.cpp +++ b/velox/exec/tests/utils/TpchQueryBuilder.cpp @@ -26,9 +26,7 @@ namespace facebook::velox::exec::test { namespace { int64_t toDate(std::string_view stringDate) { - Date date; - parseTo(stringDate, date); - return date.days(); + return DATE()->toDays(stringDate); } /// DWRF does not support Date type and Varchar is used. diff --git a/velox/expression/CastExpr.cpp b/velox/expression/CastExpr.cpp index a89a329e740..877ffeb8ccc 100644 --- a/velox/expression/CastExpr.cpp +++ b/velox/expression/CastExpr.cpp @@ -36,6 +36,17 @@ namespace facebook::velox::exec { namespace { +std::string makeErrorMessage( + const BaseVector& input, + vector_size_t row, + const TypePtr& toType) { + return fmt::format( + "Failed to cast from {} to {}: {}.", + input.type()->toString(), + toType->toString(), + input.toString(row)); +} + /// The per-row level Kernel /// @tparam ToKind The cast target type /// @tparam FromKind The expression type @@ -71,15 +82,100 @@ void applyCastKernel( } } -std::string makeErrorMessage( +VectorPtr castFromDate( + const SelectivityVector& rows, const BaseVector& input, - vector_size_t row, + exec::EvalCtx& context, const TypePtr& toType) { - return fmt::format( - "Failed to cast from {} to {}: {}.", - input.type()->toString(), - toType->toString(), - input.toString(row)); + VectorPtr castResult; + context.ensureWritable(rows, toType, castResult); + (*castResult).clearNulls(rows); + + auto* inputFlatVector = input.as>(); + switch (toType->kind()) { + case TypeKind::VARCHAR: { + auto* resultFlatVector = castResult->as>(); + context.applyToSelectedNoThrow(rows, [&](int row) { + try { + auto output = DATE()->toString(inputFlatVector->valueAt(row)); + auto writer = exec::StringWriter<>(resultFlatVector, row); + writer.resize(output.size()); + std::memcpy(writer.data(), output.data(), output.size()); + writer.finalize(); + } catch (const VeloxUserError& ue) { + VELOX_USER_FAIL( + makeErrorMessage(input, row, toType) + " " + ue.message()); + } catch (const std::exception& e) { + VELOX_USER_FAIL( + makeErrorMessage(input, row, toType) + " " + e.what()); + } + }); + return castResult; + } + case TypeKind::TIMESTAMP: { + static const int64_t kMillisPerDay{86'400'000}; + auto* resultFlatVector = castResult->as>(); + context.applyToSelectedNoThrow(rows, [&](int row) { + resultFlatVector->set( + row, + Timestamp::fromMillis( + inputFlatVector->valueAt(row) * kMillisPerDay)); + }); + return castResult; + } + default: + VELOX_UNSUPPORTED( + "Cast from DATE to {} is not supported", toType->toString()); + } +} + +VectorPtr castToDate( + const SelectivityVector& rows, + const BaseVector& input, + exec::EvalCtx& context, + const TypePtr& fromType) { + VectorPtr castResult; + context.ensureWritable(rows, DATE(), castResult); + (*castResult).clearNulls(rows); + auto* resultFlatVector = castResult->as>(); + switch (fromType->kind()) { + case TypeKind::VARCHAR: { + auto* inputVector = input.as>(); + context.applyToSelectedNoThrow(rows, [&](int row) { + try { + auto inputString = inputVector->valueAt(row); + resultFlatVector->set(row, DATE()->toDays(inputString)); + } catch (const VeloxUserError& ue) { + VELOX_USER_FAIL( + makeErrorMessage(input, row, DATE()) + " " + ue.message()); + } catch (const std::exception& e) { + VELOX_USER_FAIL( + makeErrorMessage(input, row, DATE()) + " " + e.what()); + } + }); + return castResult; + } + case TypeKind::TIMESTAMP: { + auto* inputVector = input.as>(); + static const int32_t kSecsPerDay{86'400}; + context.applyToSelectedNoThrow(rows, [&](int row) { + auto input = inputVector->valueAt(row); + auto seconds = input.getSeconds(); + if (seconds >= 0 || seconds % kSecsPerDay == 0) { + resultFlatVector->set(row, seconds / kSecsPerDay); + } else { + // For division with negatives, minus 1 to compensate the discarded + // fractional part. e.g. -1/86'400 yields 0, yet it should be + // considered as -1 day. + resultFlatVector->set(row, seconds / kSecsPerDay - 1); + } + }); + return castResult; + } + default: + VELOX_UNSUPPORTED( + "Cast from {} to DATE is not supported", fromType->toString()); + } } template @@ -228,6 +324,7 @@ void applyCastPrimitivesDispatch( input, result); } + } // namespace VectorPtr CastExpr::applyMap( @@ -522,6 +619,10 @@ void CastExpr::applyPeeled( } else { castFromOperator_->castFrom(input, context, rows, toType, result); } + } else if (fromType->isDate()) { + result = castFromDate(rows, input, context, toType); + } else if (toType->isDate()) { + result = castToDate(rows, input, context, fromType); } else if (toType->isShortDecimal()) { result = applyDecimal(rows, input, context, fromType, toType); } else if (toType->isLongDecimal()) { diff --git a/velox/expression/ConstantExpr.cpp b/velox/expression/ConstantExpr.cpp index 2647a1efae5..65f5bb14942 100644 --- a/velox/expression/ConstantExpr.cpp +++ b/velox/expression/ConstantExpr.cpp @@ -141,6 +141,14 @@ void appendSqlLiteral( return; } + if (vector.type()->isDate()) { + auto dateVector = vector.wrappedVector()->as>(); + out << "'" + << DATE()->toString(dateVector->valueAt(vector.wrappedIndex(row))) + << "'::" << vector.type()->toString(); + return; + } + switch (vector.typeKind()) { case TypeKind::BOOLEAN: { auto value = vector.as>()->valueAt(row); @@ -151,7 +159,6 @@ void appendSqlLiteral( case TypeKind::SMALLINT: case TypeKind::INTEGER: case TypeKind::BIGINT: - case TypeKind::DATE: case TypeKind::TIMESTAMP: case TypeKind::REAL: case TypeKind::DOUBLE: diff --git a/velox/expression/ExprToSubfieldFilter.cpp b/velox/expression/ExprToSubfieldFilter.cpp index be2b6b49347..4e0e92f220b 100644 --- a/velox/expression/ExprToSubfieldFilter.cpp +++ b/velox/expression/ExprToSubfieldFilter.cpp @@ -159,8 +159,6 @@ std::unique_ptr makeLessThanOrEqualFilter( return lessThanOrEqualFloat(singleValue(upper)); case TypeKind::VARCHAR: return lessThanOrEqual(singleValue(upper)); - case TypeKind::DATE: - return lessThanOrEqual(singleValue(upper).days()); default: return nullptr; } @@ -190,8 +188,6 @@ std::unique_ptr makeLessThanFilter( return lessThanFloat(singleValue(upper)); case TypeKind::VARCHAR: return lessThan(singleValue(upper)); - case TypeKind::DATE: - return lessThan(singleValue(upper).days()); default: return nullptr; } @@ -221,8 +217,6 @@ std::unique_ptr makeGreaterThanOrEqualFilter( return greaterThanOrEqualFloat(singleValue(lower)); case TypeKind::VARCHAR: return greaterThanOrEqual(singleValue(lower)); - case TypeKind::DATE: - return greaterThanOrEqual(singleValue(lower).days()); default: return nullptr; } @@ -252,8 +246,6 @@ std::unique_ptr makeGreaterThanFilter( return greaterThanFloat(singleValue(lower)); case TypeKind::VARCHAR: return greaterThan(singleValue(lower)); - case TypeKind::DATE: - return greaterThan(singleValue(lower).days()); default: return nullptr; } @@ -281,8 +273,6 @@ std::unique_ptr makeEqualFilter( return equalHugeint(singleValue(value)); case TypeKind::VARCHAR: return equal(singleValue(value)); - case TypeKind::DATE: - return equal(singleValue(value).days()); default: return nullptr; } @@ -410,6 +400,12 @@ std::unique_ptr makeBetweenFilter( return nullptr; } switch (lower->typeKind()) { + case TypeKind::INTEGER: + if (negated) { + return notBetween( + singleValue(lower), singleValue(upper)); + } + return between(singleValue(lower), singleValue(upper)); case TypeKind::BIGINT: if (negated) { return notBetween( @@ -425,13 +421,6 @@ std::unique_ptr makeBetweenFilter( return negated ? nullptr : betweenFloat(singleValue(lower), singleValue(upper)); - case TypeKind::DATE: - if (negated) { - return notBetween( - singleValue(lower).days(), singleValue(upper).days()); - } - return between( - singleValue(lower).days(), singleValue(upper).days()); case TypeKind::VARCHAR: if (negated) { return notBetween( diff --git a/velox/expression/FunctionSignature.cpp b/velox/expression/FunctionSignature.cpp index 9852182c1c4..767f8c9a273 100644 --- a/velox/expression/FunctionSignature.cpp +++ b/velox/expression/FunctionSignature.cpp @@ -167,7 +167,7 @@ void validateBaseTypeAndCollectTypeParams( if (!isPositiveInteger(typeName) && !tryMapNameToTypeKind(typeName).has_value() && - !isDecimalName(typeName)) { + !isDecimalName(typeName) && !isDateName(typeName)) { VELOX_USER_CHECK(hasType(typeName), "Type doesn't exist: {}", typeName); } diff --git a/velox/expression/UdfTypeResolver.h b/velox/expression/UdfTypeResolver.h index 12a5830288d..c0bc9003966 100644 --- a/velox/expression/UdfTypeResolver.h +++ b/velox/expression/UdfTypeResolver.h @@ -115,6 +115,13 @@ struct resolver { using out_type = StringWriter; }; +template <> +struct resolver { + using in_type = int32_t; + using null_free_in_type = in_type; + using out_type = int32_t; +}; + template <> struct resolver { using in_type = int64_t; diff --git a/velox/expression/tests/CastExprTest.cpp b/velox/expression/tests/CastExprTest.cpp index c69a1ab5c9f..15316669204 100644 --- a/velox/expression/tests/CastExprTest.cpp +++ b/velox/expression/tests/CastExprTest.cpp @@ -140,7 +140,9 @@ class CastExprTest : public functions::test::CastBaseTest { std::vector> input, std::vector> expectedResult, bool expectFailure = false, - bool tryCast = false) { + bool tryCast = false, + const TypePtr& fromType = CppToType::create(), + const TypePtr& toType = CppToType::create()) { std::vector rawInput(input.size()); for (auto index = 0; index < input.size(); index++) { if (input[index].has_value()) { @@ -148,7 +150,7 @@ class CastExprTest : public functions::test::CastBaseTest { } } // Create input vector using values and nulls - auto inputVector = makeFlatVector(rawInput); + auto inputVector = makeFlatVector(rawInput, fromType); for (auto index = 0; index < input.size(); index++) { if (!input[index].has_value()) { @@ -167,7 +169,7 @@ class CastExprTest : public functions::test::CastBaseTest { // run try cast and get the result vector auto result = evaluate(castFunction + "(c0 as " + typeString + ")", rowVector); - auto expected = makeNullableFlatVector(expectedResult); + auto expected = makeNullableFlatVector(expectedResult, toType); assertEqualVectors(expected, result); } @@ -307,12 +309,12 @@ TEST_F(CastExprTest, timestampToString) { } TEST_F(CastExprTest, dateToTimestamp) { - testCast( + testCast( "timestamp", { - Date(0), - Date(10957), - Date(14557), + 0, + 10957, + 14557, std::nullopt, }, { @@ -320,11 +322,15 @@ TEST_F(CastExprTest, dateToTimestamp) { Timestamp(946684800, 0), Timestamp(1257724800, 0), std::nullopt, - }); + }, + false, + false, + DATE(), + TIMESTAMP()); } TEST_F(CastExprTest, timestampToDate) { - testCast( + testCast( "date", { Timestamp(0, 0), @@ -333,11 +339,15 @@ TEST_F(CastExprTest, timestampToDate) { std::nullopt, }, { - Date(0), - Date(10957), - Date(14557), + 0, + 10957, + 14557, std::nullopt, - }); + }, + false, + false, + TIMESTAMP(), + DATE()); } TEST_F(CastExprTest, timestampInvalid) { @@ -401,33 +411,40 @@ TEST_F(CastExprTest, date) { "1920-01-02", std::nullopt, }; - std::vector> result{ - Date(0), - Date(18262), - Date(60577), - Date(-5), - Date(-57604), - Date(-18262), + std::vector> result{ + 0, + 18262, + 60577, + -5, + -57604, + -18262, std::nullopt, }; - testCast("date", input, result); + testCast( + "date", input, result, false, false, VARCHAR(), DATE()); setCastIntByTruncate(true); - testCast("date", input, result); + testCast( + "date", input, result, false, false, VARCHAR(), DATE()); } TEST_F(CastExprTest, invalidDate) { - testCast("date", {12}, {Date(0)}, true); - testCast("date", {1234}, {Date(0)}, true); - testCast("date", {1234}, {Date(0)}, true); - testCast("date", {1234}, {Date(0)}, true); - - testCast("date", {12.99}, {Date(0)}, true); - testCast("date", {12.99}, {Date(0)}, true); + testCast("date", {12}, {0}, true, false, TINYINT(), DATE()); + testCast( + "date", {1234}, {0}, true, false, SMALLINT(), DATE()); + testCast( + "date", {1234}, {0}, true, false, INTEGER(), DATE()); + testCast( + "date", {1234}, {0}, true, false, BIGINT(), DATE()); + + testCast("date", {12.99}, {0}, true, false, REAL(), DATE()); + testCast( + "date", {12.99}, {0}, true, false, DOUBLE(), DATE()); // Parsing an ill-formated date. - testCast("date", {"2012-Oct-23"}, {Date(0)}, true); + testCast( + "date", {"2012-Oct-23"}, {0}, true, false, VARCHAR(), DATE()); } TEST_F(CastExprTest, truncateVsRound) { diff --git a/velox/expression/tests/ExprTest.cpp b/velox/expression/tests/ExprTest.cpp index f3ebcdb720c..e174bce3464 100644 --- a/velox/expression/tests/ExprTest.cpp +++ b/velox/expression/tests/ExprTest.cpp @@ -2520,8 +2520,8 @@ TEST_F(ExprTest, constantToSql) { ASSERT_EQ(toSql(2134456LL), "'2134456'::BIGINT"); ASSERT_EQ(toSql(variant::null(TypeKind::BIGINT)), "NULL::BIGINT"); - ASSERT_EQ(toSql(Date(18'506)), "'2020-09-01'::DATE"); - ASSERT_EQ(toSql(variant::null(TypeKind::DATE)), "NULL::DATE"); + ASSERT_EQ(toSql(18'506, DATE()), "'2020-09-01'::DATE"); + ASSERT_EQ(toSql(variant::null(TypeKind::INTEGER), DATE()), "NULL::DATE"); ASSERT_EQ( toSql(Timestamp(123'456, 123'000)), diff --git a/velox/functions/lib/DateTimeFormatter.cpp b/velox/functions/lib/DateTimeFormatter.cpp index 3a104a6f1d9..bdb12da9f1a 100644 --- a/velox/functions/lib/DateTimeFormatter.cpp +++ b/velox/functions/lib/DateTimeFormatter.cpp @@ -17,7 +17,6 @@ #include "velox/functions/lib/DateTimeFormatter.h" #include #include -#include #include #include #include "velox/external/date/date.h" diff --git a/velox/functions/lib/TimeUtils.h b/velox/functions/lib/TimeUtils.h index 66412c2924d..415f2e05ae5 100644 --- a/velox/functions/lib/TimeUtils.h +++ b/velox/functions/lib/TimeUtils.h @@ -19,7 +19,6 @@ #include "velox/core/QueryConfig.h" #include "velox/external/date/tz.h" #include "velox/functions/Macros.h" -#include "velox/type/Date.h" namespace facebook::velox::functions { namespace { @@ -59,14 +58,15 @@ std::tm getDateTime(Timestamp timestamp, const date::time_zone* timeZone) { return dateTime; } +// days is the number of days since Epoch. FOLLY_ALWAYS_INLINE -std::tm getDateTime(Date date) { - int64_t seconds = date.days() * kSecondsInDay; +std::tm getDateTime(int32_t days) { + int64_t seconds = days * kSecondsInDay; std::tm dateTime; VELOX_USER_CHECK_NOT_NULL( gmtime_r((const time_t*)&seconds, &dateTime), "Date is too large: {} days", - date.days()); + days); return dateTime; } diff --git a/velox/functions/prestosql/DateTimeFunctions.h b/velox/functions/prestosql/DateTimeFunctions.h index 0ec18bd5cb1..5c8f42cc2e6 100644 --- a/velox/functions/prestosql/DateTimeFunctions.h +++ b/velox/functions/prestosql/DateTimeFunctions.h @@ -106,20 +106,35 @@ struct DateFunction : public TimestampWithTimezoneSupport { FOLLY_ALWAYS_INLINE void call( out_type& result, const arg_type& date) { - result = util::Converter::cast(date); + bool nullOutput; + result = DATE()->toDays(date); + } + + int32_t timestampToDate(const Timestamp& t, bool& nullOutput) { + static const int32_t kSecsPerDay{86'400}; + auto seconds = t.getSeconds(); + if (seconds >= 0 || seconds % kSecsPerDay == 0) { + return seconds / kSecsPerDay; + } + // For division with negatives, minus 1 to compensate the discarded + // fractional part. e.g. -1/86'400 yields 0, yet it should be considered as + // -1 day. + return seconds / kSecsPerDay - 1; } FOLLY_ALWAYS_INLINE void call( out_type& result, const arg_type& timestamp) { - result = util::Converter::cast(timestamp); + bool nullOutput; + result = timestampToDate(timestamp, nullOutput); } FOLLY_ALWAYS_INLINE void call( out_type& result, const arg_type& timestampWithTimezone) { - result = util::Converter::cast( - this->toTimestamp(timestampWithTimezone)); + bool nullOutput; + result = + timestampToDate(this->toTimestamp(timestampWithTimezone), nullOutput); } }; @@ -306,14 +321,13 @@ struct DateMinusIntervalDayTime { VELOX_DEFINE_FUNCTION_TYPES(T); FOLLY_ALWAYS_INLINE void call( - Date& result, + out_type& result, const arg_type& date, const arg_type& interval) { VELOX_USER_CHECK( isIntervalWholeDays(interval), "Cannot subtract hours, minutes, seconds or milliseconds from a date"); - result = date; - result.addDays(-intervalDays(interval)); + result = addToDate(date, DateTimeUnit::kDay, -intervalDays(interval)); } }; @@ -322,14 +336,13 @@ struct DatePlusIntervalDayTime { VELOX_DEFINE_FUNCTION_TYPES(T); FOLLY_ALWAYS_INLINE void call( - Date& result, + out_type& result, const arg_type& date, const arg_type& interval) { VELOX_USER_CHECK( isIntervalWholeDays(interval), "Cannot add hours, minutes, seconds or milliseconds to a date"); - result = date; - result.addDays(intervalDays(interval)); + result = addToDate(date, DateTimeUnit::kDay, intervalDays(interval)); } }; @@ -706,14 +719,14 @@ struct DateTruncFunction : public TimestampWithTimezoneSupport { : getDateUnit(unitString, true).value(); if (unit == DateTimeUnit::kDay) { - result = Date(date.days()); + result = date; return; } auto dateTime = getDateTime(date); adjustDateTime(dateTime, unit); - result = Date(timegm(&dateTime) / kSecondsInDay); + result = timegm(&dateTime) / kSecondsInDay; } FOLLY_ALWAYS_INLINE void call( @@ -1119,13 +1132,15 @@ struct ParseDateTimeFunction { template struct CurrentDateFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + const date::time_zone* timeZone_ = nullptr; FOLLY_ALWAYS_INLINE void initialize(const core::QueryConfig& config) { timeZone_ = getTimeZoneFromConfig(config); } - FOLLY_ALWAYS_INLINE void call(Date& result) { + FOLLY_ALWAYS_INLINE void call(out_type& result) { auto now = Timestamp::now(); if (timeZone_ != nullptr) { now.toTimezone(*timeZone_); @@ -1133,8 +1148,8 @@ struct CurrentDateFunction { const std::chrono:: time_point localTimepoint(std::chrono::milliseconds(now.toMillis())); - auto daysSinceEpoch = std::chrono::floor(localTimepoint); - result = Date(daysSinceEpoch.time_since_epoch().count()); + result = std::chrono::floor((localTimepoint).time_since_epoch()) + .count(); } }; diff --git a/velox/functions/prestosql/DateTimeImpl.h b/velox/functions/prestosql/DateTimeImpl.h index 7867659cb21..cb2b5396632 100644 --- a/velox/functions/prestosql/DateTimeImpl.h +++ b/velox/functions/prestosql/DateTimeImpl.h @@ -18,7 +18,6 @@ #include #include #include "velox/external/date/date.h" -#include "velox/type/Date.h" #include "velox/type/Timestamp.h" #include "velox/type/TimestampConversion.h" @@ -92,14 +91,15 @@ enum class DateTimeUnit { // 2020-02-29 + (1 year) = 2021-02-28 // 2021-02-28 - (1 year) = 2020-02-28 FOLLY_ALWAYS_INLINE -Date addToDate(const Date& date, const DateTimeUnit unit, const int32_t value) { +int32_t +addToDate(const int32_t input, const DateTimeUnit unit, const int32_t value) { // TODO(gaoge): Handle overflow and underflow with 64-bit representation if (value == 0) { - return date; + return input; } - const std::chrono::time_point inDate( - date::days(date.days())); + const std::chrono::time_point inDate{ + date::days(input)}; std::chrono::time_point outDate; if (unit == DateTimeUnit::kDay) { @@ -124,7 +124,7 @@ Date addToDate(const Date& date, const DateTimeUnit unit, const int32_t value) { outDate = date::sys_days{outCalDate}; } - return Date(outDate.time_since_epoch().count()); + return outDate.time_since_epoch().count(); } FOLLY_ALWAYS_INLINE Timestamp addToTimestamp( @@ -149,12 +149,12 @@ FOLLY_ALWAYS_INLINE Timestamp addToTimestamp( case DateTimeUnit::kQuarter: case DateTimeUnit::kMonth: case DateTimeUnit::kDay: { - const Date inDate( + const int32_t inDate = std::chrono::duration_cast(inTimestamp.time_since_epoch()) - .count()); - const Date outDate = addToDate(inDate, unit, value); + .count(); + const int32_t outDate = addToDate(inDate, unit, value); - outTimestamp = inTimestamp + date::days(outDate.days() - inDate.days()); + outTimestamp = inTimestamp + date::days(outDate - inDate); break; } case DateTimeUnit::kHour: { @@ -297,15 +297,17 @@ FOLLY_ALWAYS_INLINE int64_t diffTimestamp( } FOLLY_ALWAYS_INLINE -int64_t -diffDate(const DateTimeUnit unit, const Date& fromDate, const Date& toDate) { +int64_t diffDate( + const DateTimeUnit unit, + const int32_t fromDate, + const int32_t toDate) { if (fromDate == toDate) { return 0; } return diffTimestamp( unit, // prevent overflow - Timestamp((int64_t)fromDate.days() * util::kSecsPerDay, 0), - Timestamp((int64_t)toDate.days() * util::kSecsPerDay, 0)); + Timestamp((int64_t)fromDate * util::kSecsPerDay, 0), + Timestamp((int64_t)toDate * util::kSecsPerDay, 0)); } } // namespace facebook::velox::functions diff --git a/velox/functions/prestosql/GreatestLeast.cpp b/velox/functions/prestosql/GreatestLeast.cpp index f65dcecac38..a89cdd26c6a 100644 --- a/velox/functions/prestosql/GreatestLeast.cpp +++ b/velox/functions/prestosql/GreatestLeast.cpp @@ -148,10 +148,6 @@ class ExtremeValueFunction : public exec::VectorFunction { applyTyped::NativeType>( rows, args, outputType, context, result); return; - case TypeKind::DATE: - applyTyped::NativeType>( - rows, args, outputType, context, result); - return; default: VELOX_FAIL( "Unsupported input type for {}: {}", diff --git a/velox/functions/prestosql/InPredicate.cpp b/velox/functions/prestosql/InPredicate.cpp index f59940bfb52..d1360b1bfd9 100644 --- a/velox/functions/prestosql/InPredicate.cpp +++ b/velox/functions/prestosql/InPredicate.cpp @@ -113,29 +113,6 @@ std::pair, bool> createBytesValuesFilter( return {std::make_unique(values, nullAllowed), false}; } -std::pair, bool> createDateValuesFilter( - const std::vector& inputArgs) { - auto valuesPair = toValues(inputArgs); - if (!valuesPair.has_value()) { - return {nullptr, false}; - } - - const auto& values = valuesPair.value().first; - bool nullAllowed = valuesPair.value().second; - - if (values.empty() && nullAllowed) { - return {nullptr, true}; - } - VELOX_USER_CHECK( - !values.empty(), - "IN predicate expects at least one non-null value in the in-list"); - std::vector dayValues; - for (auto date : values) { - dayValues.push_back(date.days()); - } - return {common::createBigintValues(dayValues, nullAllowed), false}; -} - class InPredicate : public exec::VectorFunction { public: explicit InPredicate(std::unique_ptr filter, bool alwaysNull) @@ -166,9 +143,6 @@ class InPredicate : public exec::VectorFunction { case TypeKind::VARBINARY: filter = createBytesValuesFilter(inputArgs); break; - case TypeKind::DATE: - filter = createDateValuesFilter(inputArgs); - break; case TypeKind::UNKNOWN: filter = {nullptr, true}; break; @@ -222,11 +196,6 @@ class InPredicate : public exec::VectorFunction { return filter_->testBytes(value.data(), value.size()); }); break; - case TypeKind::DATE: - applyTyped(rows, input, context, result, [&](Date value) { - return filter_->testInt64(value.days()); - }); - break; default: VELOX_UNSUPPORTED( "Unsupported input type for the IN predicate: {}", diff --git a/velox/functions/prestosql/Sequence.cpp b/velox/functions/prestosql/Sequence.cpp index 6c7fbf6e1e4..40c144adf83 100644 --- a/velox/functions/prestosql/Sequence.cpp +++ b/velox/functions/prestosql/Sequence.cpp @@ -32,8 +32,8 @@ int64_t toInt64(int64_t value) { } template <> -int64_t toInt64(Date value) { - return value.days(); +int64_t toInt64(int32_t value) { + return value; } template <> @@ -52,8 +52,8 @@ int64_t add(int64_t value, int64_t steps) { } template <> -Date add(Date value, Days steps) { - return Date(value.days() + steps); +int32_t add(int32_t value, int64_t steps) { + return value + steps; } template <> @@ -62,7 +62,7 @@ Timestamp add(Timestamp value, int64_t steps) { } template <> -Date add(Date value, Months steps) { +int32_t add(int32_t value, Months steps) { return addToDate(value, DateTimeUnit::kMonth, steps); } @@ -76,7 +76,7 @@ int128_t getStepCount(T start, T end, int32_t step) { VELOX_FAIL("Unexpected start/end type for argument INTERVAL_YEAR_MONTH"); } -int128_t getStepCount(Date start, Date end, int32_t step) { +int128_t getStepCount(int32_t start, int32_t end, int32_t step) { return diffDate(DateTimeUnit::kMonth, start, end) / step + 1; } @@ -276,14 +276,16 @@ std::vector> signatures() { std::shared_ptr create( const std::string& /* name */, const std::vector& inputArgs) { + if (inputArgs[0].type->isDate()) { + if (inputArgs.size() > 2 && inputArgs[2].type->isIntervalYearMonth()) { + return std::make_shared>(); + } + return std::make_shared>(); + } + switch (inputArgs[0].type->kind()) { case TypeKind::BIGINT: return std::make_shared>(); - case TypeKind::DATE: - if (inputArgs.size() > 2 && inputArgs[2].type->isIntervalYearMonth()) { - return std::make_shared>(); - } - return std::make_shared>(); case TypeKind::TIMESTAMP: if (inputArgs.size() > 2 && inputArgs[2].type->isIntervalYearMonth()) { return std::make_shared>(); diff --git a/velox/functions/prestosql/aggregates/ArbitraryAggregate.cpp b/velox/functions/prestosql/aggregates/ArbitraryAggregate.cpp index cc9b4478a06..9665923cc38 100644 --- a/velox/functions/prestosql/aggregates/ArbitraryAggregate.cpp +++ b/velox/functions/prestosql/aggregates/ArbitraryAggregate.cpp @@ -290,8 +290,6 @@ exec::AggregateRegistrationResult registerArbitrary(const std::string& name) { return std::make_unique>(inputType); case TypeKind::TIMESTAMP: return std::make_unique>(inputType); - case TypeKind::DATE: - return std::make_unique>(inputType); case TypeKind::VARCHAR: case TypeKind::ARRAY: case TypeKind::MAP: diff --git a/velox/functions/prestosql/aggregates/HistogramAggregate.cpp b/velox/functions/prestosql/aggregates/HistogramAggregate.cpp index f6e462a0625..2bad362ca99 100644 --- a/velox/functions/prestosql/aggregates/HistogramAggregate.cpp +++ b/velox/functions/prestosql/aggregates/HistogramAggregate.cpp @@ -403,8 +403,6 @@ exec::AggregateRegistrationResult registerHistogram(const std::string& name) { return std::make_unique>(resultType); case TypeKind::TIMESTAMP: return std::make_unique>(resultType); - case TypeKind::DATE: - return std::make_unique>(resultType); case TypeKind::VARCHAR: return std::make_unique>(resultType); default: diff --git a/velox/functions/prestosql/aggregates/MinMaxAggregates.cpp b/velox/functions/prestosql/aggregates/MinMaxAggregates.cpp index e8fc6e6d69f..1da6a34fd1e 100644 --- a/velox/functions/prestosql/aggregates/MinMaxAggregates.cpp +++ b/velox/functions/prestosql/aggregates/MinMaxAggregates.cpp @@ -31,21 +31,9 @@ namespace { template struct MinMaxTrait : public std::numeric_limits {}; -template <> -struct MinMaxTrait { - static constexpr Date lowest() { - return Date(std::numeric_limits::lowest()); - } - - static constexpr Date max() { - return Date(std::numeric_limits::max()); - } -}; - template class MinMaxAggregate : public SimpleNumericAggregate { using BaseAggregate = SimpleNumericAggregate; - public: explicit MinMaxAggregate(TypePtr resultType) : BaseAggregate(resultType) {} @@ -491,8 +479,6 @@ exec::AggregateRegistrationResult registerMinMax(const std::string& name) { return std::make_unique>(resultType); case TypeKind::TIMESTAMP: return std::make_unique>(resultType); - case TypeKind::DATE: - return std::make_unique>(resultType); case TypeKind::HUGEINT: return std::make_unique>(resultType); case TypeKind::VARCHAR: diff --git a/velox/functions/prestosql/aggregates/MinMaxByAggregates.cpp b/velox/functions/prestosql/aggregates/MinMaxByAggregates.cpp index a2894b5fff2..524c16aa94f 100644 --- a/velox/functions/prestosql/aggregates/MinMaxByAggregates.cpp +++ b/velox/functions/prestosql/aggregates/MinMaxByAggregates.cpp @@ -125,17 +125,6 @@ struct AccumulatorTypeTraits(), void>> { template struct MinMaxTrait : public std::numeric_limits {}; -template <> -struct MinMaxTrait { - static constexpr Date lowest() { - return Date(std::numeric_limits::min()); - } - - static constexpr Date max() { - return Date(std::numeric_limits::max()); - } -}; - /// MinMaxByAggregate is the base class for min_by and max_by functions /// with numeric value and comparison types. These functions return the value of /// X associated with the minimum/maximum value of Y over all input values. @@ -1097,8 +1086,6 @@ std::unique_ptr create( return std::make_unique>(resultType); case TypeKind::VARCHAR: return std::make_unique>(resultType); - case TypeKind::DATE: - return std::make_unique>(resultType); case TypeKind::TIMESTAMP: return std::make_unique>(resultType); default: @@ -1131,8 +1118,6 @@ std::unique_ptr create( case TypeKind::VARCHAR: return create( resultType, compareType, errorMessage); - case TypeKind::DATE: - return create(resultType, compareType, errorMessage); case TypeKind::TIMESTAMP: return create( resultType, compareType, errorMessage); diff --git a/velox/functions/prestosql/aggregates/PrestoHasher.cpp b/velox/functions/prestosql/aggregates/PrestoHasher.cpp index b18d341d21b..a2e9d93ac9b 100644 --- a/velox/functions/prestosql/aggregates/PrestoHasher.cpp +++ b/velox/functions/prestosql/aggregates/PrestoHasher.cpp @@ -99,15 +99,6 @@ FOLLY_ALWAYS_INLINE void PrestoHasher::hash( }); } -template <> -FOLLY_ALWAYS_INLINE void PrestoHasher::hash( - const SelectivityVector& rows, - BufferPtr& hashes) { - applyHashFunction(rows, *vector_.get(), hashes, [&](auto row) { - return hashInteger(vector_->valueAt(row).days()); - }); -} - template <> FOLLY_ALWAYS_INLINE void PrestoHasher::hash( const SelectivityVector& rows, diff --git a/velox/functions/prestosql/aggregates/tests/ArbitraryTest.cpp b/velox/functions/prestosql/aggregates/tests/ArbitraryTest.cpp index 83bbd7c6a7a..494c0c986be 100644 --- a/velox/functions/prestosql/aggregates/tests/ArbitraryTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/ArbitraryTest.cpp @@ -281,22 +281,15 @@ TEST_F(ArbitraryTest, date) { // Grouping key. makeFlatVector({1, 1, 2, 2, 3, 3, 4, 4}), // Input values: constant within groups. - makeNullableFlatVector( - {Date(125), - Date(125), - Date(126), - Date(126), - std::nullopt, - std::nullopt, - std::nullopt, - Date(128)}), + makeNullableFlatVector( + {125, 125, 126, 126, std::nullopt, std::nullopt, std::nullopt, 128}, + DATE()), makeConstant(std::nullopt, 8), }); auto expectedResult = makeRowVector({ makeFlatVector({1, 2, 3, 4}), - makeNullableFlatVector( - {Date(125), Date(126), std::nullopt, Date(128)}), + makeNullableFlatVector({125, 126, std::nullopt, 128}, DATE()), }); testAggregations({data}, {"c0"}, {"arbitrary(c1)"}, {expectedResult}); @@ -309,10 +302,10 @@ TEST_F(ArbitraryTest, date) { auto result = readSingleValue(plan); ASSERT_TRUE(!result.isNull()); - ASSERT_EQ(result.kind(), TypeKind::DATE); + ASSERT_EQ(result.kind(), TypeKind::INTEGER); - auto date = result.value(); - ASSERT_EQ(date, Date(125)); + auto date = result.value(); + ASSERT_EQ(date, 125); testAggregations({data}, {}, {"arbitrary(c2)"}, "SELECT null"); } diff --git a/velox/functions/prestosql/aggregates/tests/ChecksumAggregateTest.cpp b/velox/functions/prestosql/aggregates/tests/ChecksumAggregateTest.cpp index a252365c874..2d09330b130 100644 --- a/velox/functions/prestosql/aggregates/tests/ChecksumAggregateTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/ChecksumAggregateTest.cpp @@ -34,8 +34,9 @@ class ChecksumAggregateTest : public AggregationTestBase { template void assertSingleGroupChecksum( const std::vector>& data, - const std::string& checksum) { - auto inputVector = makeNullableFlatVector(data); + const std::string& checksum, + const TypePtr& type = CppToType::create()) { + auto inputVector = makeNullableFlatVector(data, type); assertChecksum(inputVector, checksum); } @@ -138,9 +139,9 @@ TEST_F(ChecksumAggregateTest, reals) { } TEST_F(ChecksumAggregateTest, dates) { - assertSingleGroupChecksum({Date(0)}, "AAAAAAAAAAA="); - assertSingleGroupChecksum({Date(1)}, "vmaSXOnPGBc="); - assertSingleGroupChecksum({{}}, "h8rrhbF5N54="); + assertSingleGroupChecksum({0}, "AAAAAAAAAAA=", DATE()); + assertSingleGroupChecksum({1}, "vmaSXOnPGBc=", DATE()); + assertSingleGroupChecksum({{}}, "h8rrhbF5N54=", DATE()); } TEST_F(ChecksumAggregateTest, timestamps) { diff --git a/velox/functions/prestosql/aggregates/tests/HistogramTest.cpp b/velox/functions/prestosql/aggregates/tests/HistogramTest.cpp index 427d0a09a38..5fe6cdd399f 100644 --- a/velox/functions/prestosql/aggregates/tests/HistogramTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/HistogramTest.cpp @@ -166,16 +166,14 @@ TEST_F(HistogramTest, groupByDate) { auto vector1 = makeFlatVector( num, [](vector_size_t row) { return row % 3; }, nullEvery(4)); - auto vector2 = makeFlatVector( - num, [](vector_size_t row) { return Date{row % 2}; }, nullEvery(5)); + auto vector2 = makeFlatVector( + num, [](vector_size_t row) { return row % 2; }, nullEvery(5), DATE()); auto expected = makeRowVector( {makeNullableFlatVector({std::nullopt, 0, 1, 2}), - makeMapVector( - {{{Date{0}, 2}}, - {{Date{0}, 1}, {Date{1}, 2}}, - {{Date{1}, 2}}, - {{Date{0}, 1}}})}); + makeMapVector( + {{{{0}, 2}}, {{{0}, 1}, {{1}, 2}}, {{{1}, 2}}, {{{0}, 1}}}, + MAP(DATE(), BIGINT()))}); testHistogram("histogram(c1)", {"c0"}, vector1, vector2, expected); } @@ -252,11 +250,11 @@ TEST_F(HistogramTest, globalTimestamp) { TEST_F(HistogramTest, globalDate) { vector_size_t num = 10; - auto vector = makeFlatVector( - num, [](vector_size_t row) { return Date{row % 4}; }, nullEvery(7)); + auto vector = makeFlatVector( + num, [](vector_size_t row) { return row % 4; }, nullEvery(7), DATE()); - auto expected = makeRowVector({makeMapVector( - {{{Date{0}, 2}, {Date{1}, 3}, {Date{2}, 2}, {Date{3}, 1}}})}); + auto expected = makeRowVector({makeMapVector( + {{{{0}, 2}, {{1}, 3}, {{2}, 2}, {{3}, 1}}}, MAP(DATE(), BIGINT()))}); testHistogram("histogram(c1)", {}, vector, vector, expected); } diff --git a/velox/functions/prestosql/aggregates/tests/MaxSizeForStatsTest.cpp b/velox/functions/prestosql/aggregates/tests/MaxSizeForStatsTest.cpp index 09600ee7369..c4c517210d3 100644 --- a/velox/functions/prestosql/aggregates/tests/MaxSizeForStatsTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/MaxSizeForStatsTest.cpp @@ -40,7 +40,7 @@ TEST_F(MaxSizeForStatsTest, nullValues) { makeNullableFlatVector({std::nullopt, std::nullopt}), makeNullableFlatVector({std::nullopt, std::nullopt}), makeNullableFlatVector({std::nullopt, std::nullopt}), - makeNullableFlatVector({std::nullopt, std::nullopt}), + makeNullableFlatVector({std::nullopt, std::nullopt}, DATE()), makeNullableFlatVector({std::nullopt, std::nullopt}), makeNullableFlatVector({std::nullopt, std::nullopt}), })}; @@ -70,7 +70,7 @@ TEST_F(MaxSizeForStatsTest, nullAndNonNullValues) { makeNullableFlatVector({std::nullopt, 0}), makeNullableFlatVector({std::nullopt, 0}), makeNullableFlatVector({std::nullopt, 0}), - makeNullableFlatVector({std::nullopt, 0}), + makeNullableFlatVector({std::nullopt, 0}, DATE()), makeNullableFlatVector({std::nullopt, Timestamp(0, 0)}), makeNullableFlatVector({std::nullopt, "std::nullopt"}), })}; @@ -109,7 +109,7 @@ TEST_F(MaxSizeForStatsTest, allScalarTypes) { makeFlatVector(4, generator), makeFlatVector(4, generator), makeFlatVector(4, generator), - makeFlatVector(4, generator), + makeFlatVector(4, generator, nullptr, DATE()), makeFlatVector(4, generator)})}; // With grouping keys. diff --git a/velox/functions/prestosql/aggregates/tests/MinMaxByAggregationTest.cpp b/velox/functions/prestosql/aggregates/tests/MinMaxByAggregationTest.cpp index 090a83474a5..610fa2dbac1 100644 --- a/velox/functions/prestosql/aggregates/tests/MinMaxByAggregationTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/MinMaxByAggregationTest.cpp @@ -44,7 +44,6 @@ const std::unordered_set kSupportedTypes = { TypeKind::REAL, TypeKind::DOUBLE, TypeKind::VARCHAR, - TypeKind::DATE, TypeKind::TIMESTAMP}; std::vector getTestParams() { @@ -81,9 +80,6 @@ std::vector getTestParams() { case TypeKind::VARCHAR: \ testFunc(); \ break; \ - case TypeKind::DATE: \ - testFunc(); \ - break; \ case TypeKind::TIMESTAMP: \ testFunc(); \ break; \ @@ -117,9 +113,6 @@ std::vector getTestParams() { case TypeKind::VARCHAR: \ EXECUTE_TEST_BY_VALUE_TYPE(testFunc, StringView); \ break; \ - case TypeKind::DATE: \ - EXECUTE_TEST_BY_VALUE_TYPE(testFunc, Date); \ - break; \ case TypeKind::TIMESTAMP: \ EXECUTE_TEST_BY_VALUE_TYPE(testFunc, Timestamp); \ break; \ @@ -257,8 +250,6 @@ VectorPtr MinMaxByAggregationTestBase::buildDataVector( return buildDataVector(size, values); case TypeKind::VARCHAR: return buildDataVector(size, values); - case TypeKind::DATE: - return buildDataVector(size, values); case TypeKind::TIMESTAMP: return buildDataVector(size, values); default: @@ -301,9 +292,6 @@ void MinMaxByAggregationTestBase::SetUp() { case TypeKind::DOUBLE: dataVectorsByType_.emplace(type, buildDataVector(numValues_)); break; - case TypeKind::DATE: - dataVectorsByType_.emplace(type, buildDataVector(numValues_)); - break; case TypeKind::TIMESTAMP: dataVectorsByType_.emplace( type, buildDataVector(numValues_)); diff --git a/velox/functions/prestosql/aggregates/tests/MinMaxTest.cpp b/velox/functions/prestosql/aggregates/tests/MinMaxTest.cpp index 758962ef831..3b947fc3b8e 100644 --- a/velox/functions/prestosql/aggregates/tests/MinMaxTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/MinMaxTest.cpp @@ -207,7 +207,7 @@ TEST_F(MinMaxTest, minMaxTimestamp) { TEST_F(MinMaxTest, largeValuesDate) { auto vectors = {makeRowVector( - {makeConstant(Date(60577), 100), makeConstant(Date(-57604), 100)})}; + {makeConstant(60577, 100, DATE()), makeConstant(-57604, 100, DATE())})}; createDuckDbTable(vectors); testAggregations( diff --git a/velox/functions/prestosql/aggregates/tests/PrestoHasherTest.cpp b/velox/functions/prestosql/aggregates/tests/PrestoHasherTest.cpp index e2e85d7e112..ef1e58a442c 100644 --- a/velox/functions/prestosql/aggregates/tests/PrestoHasherTest.cpp +++ b/velox/functions/prestosql/aggregates/tests/PrestoHasherTest.cpp @@ -171,8 +171,8 @@ TEST_F(PrestoHasherTest, timestamp) { } TEST_F(PrestoHasherTest, date) { - assertHash( - {Date(0), Date(1000), std::nullopt}, {0, 2343331593029422743, 0}); + assertHash( + {0, 1000, std::nullopt}, {0, 2343331593029422743, 0}, DATE()); } TEST_F(PrestoHasherTest, unscaledShortDecimal) { diff --git a/velox/functions/prestosql/tests/ArrayExceptTest.cpp b/velox/functions/prestosql/tests/ArrayExceptTest.cpp index 94b50921dfe..9fa40a3093b 100644 --- a/velox/functions/prestosql/tests/ArrayExceptTest.cpp +++ b/velox/functions/prestosql/tests/ArrayExceptTest.cpp @@ -257,7 +257,8 @@ TEST_F(ArrayExceptTest, varbinary) { auto right = makeNullableArrayVector( {{"b"_sv, "d"_sv}}, ARRAY(VARBINARY())); - auto expected = makeNullableArrayVector({{}}, ARRAY(VARBINARY())); + const std::vector>> data = {{}}; + auto expected = makeNullableArrayVector(data, ARRAY(VARBINARY())); testExpr(expected, "array_except(c0, c1)", {left, left}); testExpr(expected, "array_except(c0, c1)", {right, right}); diff --git a/velox/functions/prestosql/tests/ComparisonsTest.cpp b/velox/functions/prestosql/tests/ComparisonsTest.cpp index fffb94d3ca1..7c1871b4368 100644 --- a/velox/functions/prestosql/tests/ComparisonsTest.cpp +++ b/velox/functions/prestosql/tests/ComparisonsTest.cpp @@ -90,11 +90,10 @@ TEST_F(ComparisonsTest, betweenVarchar) { TEST_F(ComparisonsTest, betweenDate) { auto parseDate = [](const std::string& dateStr) { - Date returnDate; - parseTo(dateStr, returnDate); - return returnDate; + return DATE()->toDays(dateStr); }; - std::vector> testData = { + + std::vector> testData = { {parseDate("2019-05-01"), false}, {parseDate("2019-06-01"), true}, {parseDate("2019-07-01"), true}, @@ -104,7 +103,7 @@ TEST_F(ComparisonsTest, betweenDate) { auto result = evaluate>( "c0 between cast(\'2019-06-01\' as date) and cast(\'2020-06-01\' as date)", - makeRowVector({makeFlatVector(testData)})); + makeRowVector({makeFlatVector(testData, DATE())})); for (int i = 0; i < testData.size(); ++i) { EXPECT_EQ(result->valueAt(i), std::get<1>(testData[i])) << "at " << i; diff --git a/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp b/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp index f517604136e..8fb821238f8 100644 --- a/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp +++ b/velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp @@ -193,10 +193,8 @@ class DateTimeFunctionsTest : public functions::test::FunctionBaseTest { timestamp.value(), timeZoneName.value().c_str())})); } - Date parseDate(const std::string& dateStr) { - Date returnDate; - parseTo(dateStr, returnDate); - return returnDate; + int32_t parseDate(const std::string& dateStr) { + return DATE()->toDays(dateStr); } RowVectorPtr makeTimestampWithTimeZoneVector( @@ -226,7 +224,7 @@ class DateTimeFunctionsTest : public functions::test::FunctionBaseTest { std::vector({timestamps, timezones})); } - Date getCurrentDate(const std::optional& timeZone) { + int32_t getCurrentDate(const std::optional& timeZone) { return parseDate(date::format( "%Y-%m-%d", timeZone.has_value() @@ -442,14 +440,14 @@ TEST_F(DateTimeFunctionsTest, year) { } TEST_F(DateTimeFunctionsTest, yearDate) { - const auto year = [&](std::optional date) { - return evaluateOnce("year(c0)", date); + const auto year = [&](std::optional date) { + return evaluateOnce("year(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, year(std::nullopt)); - EXPECT_EQ(1970, year(Date(0))); - EXPECT_EQ(1969, year(Date(-1))); - EXPECT_EQ(2020, year(Date(18262))); - EXPECT_EQ(1920, year(Date(-18262))); + EXPECT_EQ(1970, year(DATE()->toDays("1970-01-01"))); + EXPECT_EQ(1969, year(DATE()->toDays("1969-12-31"))); + EXPECT_EQ(2020, year(DATE()->toDays("2020-01-01"))); + EXPECT_EQ(1920, year(DATE()->toDays("1920-12-31"))); } TEST_F(DateTimeFunctionsTest, yearTimestampWithTimezone) { @@ -484,8 +482,11 @@ TEST_F(DateTimeFunctionsTest, yearTimestampWithTimezone) { TEST_F(DateTimeFunctionsTest, weekDate) { const auto weekDate = [&](const char* dateString) { auto date = std::make_optional(parseDate(dateString)); - auto week = evaluateOnce("week(c0)", date).value(); - auto weekOfYear = evaluateOnce("week_of_year(c0)", date).value(); + auto week = + evaluateOnce("week(c0)", {date}, {DATE()}).value(); + auto weekOfYear = + evaluateOnce("week_of_year(c0)", {date}, {DATE()}) + .value(); VELOX_CHECK_EQ( week, weekOfYear, "week and week_of_year must return the same value"); return week; @@ -575,17 +576,17 @@ TEST_F(DateTimeFunctionsTest, quarter) { } TEST_F(DateTimeFunctionsTest, quarterDate) { - const auto quarter = [&](std::optional date) { - return evaluateOnce("quarter(c0)", date); + const auto quarter = [&](std::optional date) { + return evaluateOnce("quarter(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, quarter(std::nullopt)); - EXPECT_EQ(1, quarter(Date(0))); - EXPECT_EQ(4, quarter(Date(-1))); - EXPECT_EQ(4, quarter(Date(-40))); - EXPECT_EQ(2, quarter(Date(110))); - EXPECT_EQ(3, quarter(Date(200))); - EXPECT_EQ(1, quarter(Date(18262))); - EXPECT_EQ(1, quarter(Date(-18262))); + EXPECT_EQ(1, quarter(0)); + EXPECT_EQ(4, quarter(-1)); + EXPECT_EQ(4, quarter(-40)); + EXPECT_EQ(2, quarter(110)); + EXPECT_EQ(3, quarter(200)); + EXPECT_EQ(1, quarter(18262)); + EXPECT_EQ(1, quarter(-18262)); } TEST_F(DateTimeFunctionsTest, quarterTimestampWithTimezone) { @@ -641,16 +642,16 @@ TEST_F(DateTimeFunctionsTest, month) { } TEST_F(DateTimeFunctionsTest, monthDate) { - const auto month = [&](std::optional date) { - return evaluateOnce("month(c0)", date); + const auto month = [&](std::optional date) { + return evaluateOnce("month(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, month(std::nullopt)); - EXPECT_EQ(1, month(Date(0))); - EXPECT_EQ(12, month(Date(-1))); - EXPECT_EQ(11, month(Date(-40))); - EXPECT_EQ(2, month(Date(40))); - EXPECT_EQ(1, month(Date(18262))); - EXPECT_EQ(1, month(Date(-18262))); + EXPECT_EQ(1, month(0)); + EXPECT_EQ(12, month(-1)); + EXPECT_EQ(11, month(-40)); + EXPECT_EQ(2, month(40)); + EXPECT_EQ(1, month(18262)); + EXPECT_EQ(1, month(-18262)); } TEST_F(DateTimeFunctionsTest, monthTimestampWithTimezone) { @@ -745,16 +746,16 @@ TEST_F(DateTimeFunctionsTest, hourTimestampWithTimezone) { } TEST_F(DateTimeFunctionsTest, hourDate) { - const auto hour = [&](std::optional date) { - return evaluateOnce("hour(c0)", date); + const auto hour = [&](std::optional date) { + return evaluateOnce("hour(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, hour(std::nullopt)); - EXPECT_EQ(0, hour(Date(0))); - EXPECT_EQ(0, hour(Date(-1))); - EXPECT_EQ(0, hour(Date(-40))); - EXPECT_EQ(0, hour(Date(40))); - EXPECT_EQ(0, hour(Date(18262))); - EXPECT_EQ(0, hour(Date(-18262))); + EXPECT_EQ(0, hour(0)); + EXPECT_EQ(0, hour(-1)); + EXPECT_EQ(0, hour(-40)); + EXPECT_EQ(0, hour(40)); + EXPECT_EQ(0, hour(18262)); + EXPECT_EQ(0, hour(-18262)); } TEST_F(DateTimeFunctionsTest, dayOfMonth) { @@ -781,34 +782,34 @@ TEST_F(DateTimeFunctionsTest, dayOfMonth) { } TEST_F(DateTimeFunctionsTest, dayOfMonthDate) { - const auto day = [&](std::optional date) { - return evaluateOnce("day_of_month(c0)", date); + const auto day = [&](std::optional date) { + return evaluateOnce("day_of_month(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, day(std::nullopt)); - EXPECT_EQ(1, day(Date(0))); - EXPECT_EQ(31, day(Date(-1))); - EXPECT_EQ(22, day(Date(-40))); - EXPECT_EQ(10, day(Date(40))); - EXPECT_EQ(1, day(Date(18262))); - EXPECT_EQ(2, day(Date(-18262))); + EXPECT_EQ(1, day(0)); + EXPECT_EQ(31, day(-1)); + EXPECT_EQ(22, day(-40)); + EXPECT_EQ(10, day(40)); + EXPECT_EQ(1, day(18262)); + EXPECT_EQ(2, day(-18262)); } TEST_F(DateTimeFunctionsTest, plusMinusDateIntervalDayTime) { - const auto plus = [&](std::optional date, + const auto plus = [&](std::optional date, std::optional interval) { - return evaluateOnce( + return evaluateOnce( "c0 + c1", makeRowVector({ - makeNullableFlatVector({date}), + makeNullableFlatVector({date}, DATE()), makeNullableFlatVector({interval}, INTERVAL_DAY_TIME()), })); }; - const auto minus = [&](std::optional date, + const auto minus = [&](std::optional date, std::optional interval) { - return evaluateOnce( + return evaluateOnce( "c0 - c1", makeRowVector({ - makeNullableFlatVector({date}), + makeNullableFlatVector({date}, DATE()), makeNullableFlatVector({interval}, INTERVAL_DAY_TIME()), })); }; @@ -816,18 +817,18 @@ TEST_F(DateTimeFunctionsTest, plusMinusDateIntervalDayTime) { const int64_t oneDay(kMillisInDay * 1); const int64_t tenDays(kMillisInDay * 10); const int64_t partDay(kMillisInHour * 25); - const Date baseDate(20000); - const Date baseDatePlus1(20000 + 1); - const Date baseDatePlus10(20000 + 10); - const Date baseDateMinus1(20000 - 1); - const Date baseDateMinus10(20000 - 10); + const int32_t baseDate(20000); + const int32_t baseDatePlus1(20000 + 1); + const int32_t baseDatePlus10(20000 + 10); + const int32_t baseDateMinus1(20000 - 1); + const int32_t baseDateMinus10(20000 - 10); EXPECT_EQ(std::nullopt, plus(std::nullopt, oneDay)); - EXPECT_EQ(std::nullopt, plus(Date(10000), std::nullopt)); + EXPECT_EQ(std::nullopt, plus(10000, std::nullopt)); EXPECT_EQ(baseDatePlus1, plus(baseDate, oneDay)); EXPECT_EQ(baseDatePlus10, plus(baseDate, tenDays)); EXPECT_EQ(std::nullopt, minus(std::nullopt, oneDay)); - EXPECT_EQ(std::nullopt, minus(Date(10000), std::nullopt)); + EXPECT_EQ(std::nullopt, minus(10000, std::nullopt)); EXPECT_EQ(baseDateMinus1, minus(baseDate, oneDay)); EXPECT_EQ(baseDateMinus10, minus(baseDate, tenDays)); @@ -896,16 +897,16 @@ TEST_F(DateTimeFunctionsTest, dayOfWeek) { } TEST_F(DateTimeFunctionsTest, dayOfWeekDate) { - const auto day = [&](std::optional date) { - return evaluateOnce("day_of_week(c0)", date); + const auto day = [&](std::optional date) { + return evaluateOnce("day_of_week(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, day(std::nullopt)); - EXPECT_EQ(4, day(Date(0))); - EXPECT_EQ(3, day(Date(-1))); - EXPECT_EQ(6, day(Date(-40))); - EXPECT_EQ(2, day(Date(40))); - EXPECT_EQ(3, day(Date(18262))); - EXPECT_EQ(5, day(Date(-18262))); + EXPECT_EQ(4, day(0)); + EXPECT_EQ(3, day(-1)); + EXPECT_EQ(6, day(-40)); + EXPECT_EQ(2, day(40)); + EXPECT_EQ(3, day(18262)); + EXPECT_EQ(5, day(-18262)); } TEST_F(DateTimeFunctionsTest, dayOfWeekTimestampWithTimezone) { @@ -963,16 +964,16 @@ TEST_F(DateTimeFunctionsTest, dayOfYear) { } TEST_F(DateTimeFunctionsTest, dayOfYearDate) { - const auto day = [&](std::optional date) { - return evaluateOnce("day_of_year(c0)", date); + const auto day = [&](std::optional date) { + return evaluateOnce("day_of_year(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, day(std::nullopt)); - EXPECT_EQ(1, day(Date(0))); - EXPECT_EQ(365, day(Date(-1))); - EXPECT_EQ(326, day(Date(-40))); - EXPECT_EQ(41, day(Date(40))); - EXPECT_EQ(1, day(Date(18262))); - EXPECT_EQ(2, day(Date(-18262))); + EXPECT_EQ(1, day(0)); + EXPECT_EQ(365, day(-1)); + EXPECT_EQ(326, day(-40)); + EXPECT_EQ(41, day(40)); + EXPECT_EQ(1, day(18262)); + EXPECT_EQ(2, day(-18262)); } TEST_F(DateTimeFunctionsTest, dayOfYearTimestampWithTimezone) { @@ -1034,18 +1035,18 @@ TEST_F(DateTimeFunctionsTest, yearOfWeek) { } TEST_F(DateTimeFunctionsTest, yearOfWeekDate) { - const auto yow = [&](std::optional date) { - return evaluateOnce("year_of_week(c0)", date); + const auto yow = [&](std::optional date) { + return evaluateOnce("year_of_week(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, yow(std::nullopt)); - EXPECT_EQ(1970, yow(Date(0))); - EXPECT_EQ(1970, yow(Date(-1))); - EXPECT_EQ(1969, yow(Date(-4))); - EXPECT_EQ(1970, yow(Date(-3))); - EXPECT_EQ(1970, yow(Date(365))); - EXPECT_EQ(1970, yow(Date(367))); - EXPECT_EQ(1971, yow(Date(368))); - EXPECT_EQ(2021, yow(Date(18900))); + EXPECT_EQ(1970, yow(0)); + EXPECT_EQ(1970, yow(-1)); + EXPECT_EQ(1969, yow(-4)); + EXPECT_EQ(1970, yow(-3)); + EXPECT_EQ(1970, yow(365)); + EXPECT_EQ(1970, yow(367)); + EXPECT_EQ(1971, yow(368)); + EXPECT_EQ(2021, yow(18900)); } TEST_F(DateTimeFunctionsTest, yearOfWeekTimestampWithTimezone) { @@ -1103,16 +1104,16 @@ TEST_F(DateTimeFunctionsTest, minute) { } TEST_F(DateTimeFunctionsTest, minuteDate) { - const auto minute = [&](std::optional date) { - return evaluateOnce("minute(c0)", date); + const auto minute = [&](std::optional date) { + return evaluateOnce("minute(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, minute(std::nullopt)); - EXPECT_EQ(0, minute(Date(0))); - EXPECT_EQ(0, minute(Date(-1))); - EXPECT_EQ(0, minute(Date(-40))); - EXPECT_EQ(0, minute(Date(40))); - EXPECT_EQ(0, minute(Date(18262))); - EXPECT_EQ(0, minute(Date(-18262))); + EXPECT_EQ(0, minute(0)); + EXPECT_EQ(0, minute(-1)); + EXPECT_EQ(0, minute(40)); + EXPECT_EQ(0, minute(40)); + EXPECT_EQ(0, minute(18262)); + EXPECT_EQ(0, minute(-18262)); } TEST_F(DateTimeFunctionsTest, minuteTimestampWithTimezone) { @@ -1167,16 +1168,16 @@ TEST_F(DateTimeFunctionsTest, second) { } TEST_F(DateTimeFunctionsTest, secondDate) { - const auto second = [&](std::optional date) { - return evaluateOnce("second(c0)", date); + const auto second = [&](std::optional date) { + return evaluateOnce("second(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, second(std::nullopt)); - EXPECT_EQ(0, second(Date(0))); - EXPECT_EQ(0, second(Date(-1))); - EXPECT_EQ(0, second(Date(-40))); - EXPECT_EQ(0, second(Date(40))); - EXPECT_EQ(0, second(Date(18262))); - EXPECT_EQ(0, second(Date(-18262))); + EXPECT_EQ(0, second(0)); + EXPECT_EQ(0, second(-1)); + EXPECT_EQ(0, second(-40)); + EXPECT_EQ(0, second(40)); + EXPECT_EQ(0, second(18262)); + EXPECT_EQ(0, second(-18262)); } TEST_F(DateTimeFunctionsTest, secondTimestampWithTimezone) { @@ -1222,16 +1223,16 @@ TEST_F(DateTimeFunctionsTest, millisecond) { } TEST_F(DateTimeFunctionsTest, millisecondDate) { - const auto millisecond = [&](std::optional date) { - return evaluateOnce("millisecond(c0)", date); + const auto millisecond = [&](std::optional date) { + return evaluateOnce("millisecond(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, millisecond(std::nullopt)); - EXPECT_EQ(0, millisecond(Date(0))); - EXPECT_EQ(0, millisecond(Date(-1))); - EXPECT_EQ(0, millisecond(Date(-40))); - EXPECT_EQ(0, millisecond(Date(40))); - EXPECT_EQ(0, millisecond(Date(18262))); - EXPECT_EQ(0, millisecond(Date(-18262))); + EXPECT_EQ(0, millisecond(0)); + EXPECT_EQ(0, millisecond(-1)); + EXPECT_EQ(0, millisecond(-40)); + EXPECT_EQ(0, millisecond(40)); + EXPECT_EQ(0, millisecond(18262)); + EXPECT_EQ(0, millisecond(-18262)); } TEST_F(DateTimeFunctionsTest, millisecondTimestampWithTimezone) { @@ -1333,37 +1334,39 @@ TEST_F(DateTimeFunctionsTest, dateTrunc) { TEST_F(DateTimeFunctionsTest, dateTruncDate) { const auto dateTrunc = [&](const std::string& unit, - std::optional date) { - return evaluateOnce(fmt::format("date_trunc('{}', c0)", unit), date); + std::optional date) { + return evaluateOnce( + fmt::format("date_trunc('{}', c0)", unit), {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, dateTrunc("year", std::nullopt)); - EXPECT_EQ(Date(0), dateTrunc("day", Date(0))); - EXPECT_EQ(Date(0), dateTrunc("year", Date(0))); - EXPECT_EQ(Date(0), dateTrunc("quarter", Date(0))); - EXPECT_EQ(Date(0), dateTrunc("month", Date(0))); - EXPECT_THROW(dateTrunc("second", Date(0)), VeloxUserError); - EXPECT_THROW(dateTrunc("minute", Date(0)), VeloxUserError); - EXPECT_THROW(dateTrunc("hour", Date(0)), VeloxUserError); + // Date(0) is 1970-01-01. + EXPECT_EQ(0, dateTrunc("day", 0)); + EXPECT_EQ(0, dateTrunc("year", 0)); + EXPECT_EQ(0, dateTrunc("quarter", 0)); + EXPECT_EQ(0, dateTrunc("month", 0)); + EXPECT_THROW(dateTrunc("second", 0), VeloxUserError); + EXPECT_THROW(dateTrunc("minute", 0), VeloxUserError); + EXPECT_THROW(dateTrunc("hour", 0), VeloxUserError); // Date(18297) is 2020-02-05. - EXPECT_EQ(Date(18297), dateTrunc("day", Date(18297))); - EXPECT_EQ(Date(18293), dateTrunc("month", Date(18297))); - EXPECT_EQ(Date(18262), dateTrunc("quarter", Date(18297))); - EXPECT_EQ(Date(18262), dateTrunc("year", Date(18297))); - EXPECT_THROW(dateTrunc("second", Date(18297)), VeloxUserError); - EXPECT_THROW(dateTrunc("minute", Date(18297)), VeloxUserError); - EXPECT_THROW(dateTrunc("hour", Date(18297)), VeloxUserError); + EXPECT_EQ(18297, dateTrunc("day", 18297)); + EXPECT_EQ(18293, dateTrunc("month", 18297)); + EXPECT_EQ(18262, dateTrunc("quarter", 18297)); + EXPECT_EQ(18262, dateTrunc("year", 18297)); + EXPECT_THROW(dateTrunc("second", 18297), VeloxUserError); + EXPECT_THROW(dateTrunc("minute", 18297), VeloxUserError); + EXPECT_THROW(dateTrunc("hour", 18297), VeloxUserError); // Date(-18297) is 1919-11-28. - EXPECT_EQ(Date(-18297), dateTrunc("day", Date(-18297))); - EXPECT_EQ(Date(-18324), dateTrunc("month", Date(-18297))); - EXPECT_EQ(Date(-18355), dateTrunc("quarter", Date(-18297))); - EXPECT_EQ(Date(-18628), dateTrunc("year", Date(-18297))); - EXPECT_THROW(dateTrunc("second", Date(-18297)), VeloxUserError); - EXPECT_THROW(dateTrunc("minute", Date(-18297)), VeloxUserError); - EXPECT_THROW(dateTrunc("hour", Date(-18297)), VeloxUserError); + EXPECT_EQ(-18297, dateTrunc("day", -18297)); + EXPECT_EQ(-18324, dateTrunc("month", -18297)); + EXPECT_EQ(-18355, dateTrunc("quarter", -18297)); + EXPECT_EQ(-18628, dateTrunc("year", -18297)); + EXPECT_THROW(dateTrunc("second", -18297), VeloxUserError); + EXPECT_THROW(dateTrunc("minute", -18297), VeloxUserError); + EXPECT_THROW(dateTrunc("hour", -18297), VeloxUserError); } TEST_F(DateTimeFunctionsTest, dateTruncTimestampWithTimezone) { @@ -1462,21 +1465,23 @@ TEST_F(DateTimeFunctionsTest, dateTruncTimestampWithTimezone) { TEST_F(DateTimeFunctionsTest, dateAddDate) { const auto dateAdd = [&](const std::string& unit, std::optional value, - std::optional date) { - return evaluateOnce( - fmt::format("date_add('{}', c0, c1)", unit), value, date); + std::optional date) { + return evaluateOnce( + fmt::format("date_add('{}', c0, c1)", unit), + {value, date}, + {INTEGER(), DATE()}); }; // Check null behaviors EXPECT_EQ(std::nullopt, dateAdd("day", 1, std::nullopt)); - EXPECT_EQ(std::nullopt, dateAdd("month", std::nullopt, Date(0))); + EXPECT_EQ(std::nullopt, dateAdd("month", std::nullopt, 0)); // Check invalid units - EXPECT_THROW(dateAdd("millisecond", 1, Date(0)), VeloxUserError); - EXPECT_THROW(dateAdd("second", 1, Date(0)), VeloxUserError); - EXPECT_THROW(dateAdd("minute", 1, Date(0)), VeloxUserError); - EXPECT_THROW(dateAdd("hour", 1, Date(0)), VeloxUserError); - EXPECT_THROW(dateAdd("invalid_unit", 1, Date(0)), VeloxUserError); + EXPECT_THROW(dateAdd("millisecond", 1, 0), VeloxUserError); + EXPECT_THROW(dateAdd("second", 1, 0), VeloxUserError); + EXPECT_THROW(dateAdd("minute", 1, 0), VeloxUserError); + EXPECT_THROW(dateAdd("hour", 1, 0), VeloxUserError); + EXPECT_THROW(dateAdd("invalid_unit", 1, 0), VeloxUserError); // Simple tests EXPECT_EQ( @@ -1794,28 +1799,28 @@ TEST_F(DateTimeFunctionsTest, dateAddTimestamp) { TEST_F(DateTimeFunctionsTest, dateDiffDate) { const auto dateDiff = [&](const std::string& unit, - std::optional date1, - std::optional date2) { - return evaluateOnce( - fmt::format("date_diff('{}', c0, c1)", unit), date1, date2); + std::optional date1, + std::optional date2) { + return evaluateOnce( + fmt::format("date_diff('{}', c0, c1)", unit), + {date1, date2}, + {DATE(), DATE()}); }; // Check null behaviors - EXPECT_EQ(std::nullopt, dateDiff("day", Date(1), std::nullopt)); - EXPECT_EQ(std::nullopt, dateDiff("month", std::nullopt, Date(0))); + EXPECT_EQ(std::nullopt, dateDiff("day", 1, std::nullopt)); + EXPECT_EQ(std::nullopt, dateDiff("month", std::nullopt, 0)); // Check invalid units VELOX_ASSERT_THROW( - dateDiff("millisecond", Date(1), Date(0)), - "millisecond is not a valid DATE field"); - VELOX_ASSERT_THROW( - dateDiff("second", Date(1), Date(0)), "second is not a valid DATE field"); + dateDiff("millisecond", 1, 0), "millisecond is not a valid DATE field"); VELOX_ASSERT_THROW( - dateDiff("minute", Date(1), Date(0)), "minute is not a valid DATE field"); + dateDiff("second", 1, 0), "second is not a valid DATE field"); VELOX_ASSERT_THROW( - dateDiff("hour", Date(1), Date(0)), "hour is not a valid DATE field"); + dateDiff("minute", 1, 0), "minute is not a valid DATE field"); + VELOX_ASSERT_THROW(dateDiff("hour", 1, 0), "hour is not a valid DATE field"); VELOX_ASSERT_THROW( - dateDiff("invalid_unit", Date(1), Date(0)), + dateDiff("invalid_unit", 1, 0), "Unsupported datetime unit: invalid_unit"); // Simple tests @@ -2916,15 +2921,15 @@ TEST_F(DateTimeFunctionsTest, dateParse) { TEST_F(DateTimeFunctionsTest, dateFunctionVarchar) { const auto dateFunction = [&](const std::optional& dateString) { - return evaluateOnce("date(c0)", dateString); + return evaluateOnce("date(c0)", dateString); }; // Date(0) is 1970-01-01. - EXPECT_EQ(Date(), dateFunction("1970-01-01")); + EXPECT_EQ(0, dateFunction("1970-01-01")); // Date(18297) is 2020-02-05. - EXPECT_EQ(Date(18297), dateFunction("2020-02-05")); + EXPECT_EQ(18297, dateFunction("2020-02-05")); // Date(-18297) is 1919-11-28. - EXPECT_EQ(Date(-18297), dateFunction("1919-11-28")); + EXPECT_EQ(-18297, dateFunction("1919-11-28")); // Illegal date format. VELOX_ASSERT_THROW( @@ -2937,45 +2942,44 @@ TEST_F(DateTimeFunctionsTest, dateFunctionTimestamp) { static const uint64_t kNanosInSecond = 1'000'000'000; const auto dateFunction = [&](std::optional timestamp) { - return evaluateOnce("date(c0)", timestamp); + return evaluateOnce("date(c0)", timestamp); }; - EXPECT_EQ(Date(), dateFunction(Timestamp())); - EXPECT_EQ(Date(1), dateFunction(Timestamp(kSecondsInDay, 0))); - EXPECT_EQ(Date(-1), dateFunction(Timestamp(-kSecondsInDay, 0))); - EXPECT_EQ(Date(18297), dateFunction(Timestamp(18297 * kSecondsInDay, 0))); - EXPECT_EQ(Date(18297), dateFunction(Timestamp(18297 * kSecondsInDay, 123))); - EXPECT_EQ(Date(-18297), dateFunction(Timestamp(-18297 * kSecondsInDay, 0))); - EXPECT_EQ(Date(-18297), dateFunction(Timestamp(-18297 * kSecondsInDay, 123))); + EXPECT_EQ(0, dateFunction(Timestamp())); + EXPECT_EQ(1, dateFunction(Timestamp(kSecondsInDay, 0))); + EXPECT_EQ(-1, dateFunction(Timestamp(-kSecondsInDay, 0))); + EXPECT_EQ(18297, dateFunction(Timestamp(18297 * kSecondsInDay, 0))); + EXPECT_EQ(18297, dateFunction(Timestamp(18297 * kSecondsInDay, 123))); + EXPECT_EQ(-18297, dateFunction(Timestamp(-18297 * kSecondsInDay, 0))); + EXPECT_EQ(-18297, dateFunction(Timestamp(-18297 * kSecondsInDay, 123))); // Last second of day 0 - EXPECT_EQ(Date(), dateFunction(Timestamp(kSecondsInDay - 1, 0))); + EXPECT_EQ(0, dateFunction(Timestamp(kSecondsInDay - 1, 0))); // Last nanosecond of day 0 - EXPECT_EQ( - Date(), dateFunction(Timestamp(kSecondsInDay - 1, kNanosInSecond - 1))); + EXPECT_EQ(0, dateFunction(Timestamp(kSecondsInDay - 1, kNanosInSecond - 1))); // Last second of day -1 - EXPECT_EQ(Date(-1), dateFunction(Timestamp(-1, 0))); + EXPECT_EQ(-1, dateFunction(Timestamp(-1, 0))); // Last nanosecond of day -1 - EXPECT_EQ(Date(-1), dateFunction(Timestamp(-1, kNanosInSecond - 1))); + EXPECT_EQ(-1, dateFunction(Timestamp(-1, kNanosInSecond - 1))); // Last second of day 18297 EXPECT_EQ( - Date(18297), + 18297, dateFunction(Timestamp(18297 * kSecondsInDay + kSecondsInDay - 1, 0))); // Last nanosecond of day 18297 EXPECT_EQ( - Date(18297), + 18297, dateFunction(Timestamp( 18297 * kSecondsInDay + kSecondsInDay - 1, kNanosInSecond - 1))); // Last second of day -18297 EXPECT_EQ( - Date(-18297), + -18297, dateFunction(Timestamp(-18297 * kSecondsInDay + kSecondsInDay - 1, 0))); // Last nanosecond of day -18297 EXPECT_EQ( - Date(-18297), + -18297, dateFunction(Timestamp( -18297 * kSecondsInDay + kSecondsInDay - 1, kNanosInSecond - 1))); } @@ -2986,90 +2990,90 @@ TEST_F(DateTimeFunctionsTest, dateFunctionTimestampWithTimezone) { const auto dateFunction = [&](std::optional timestamp, const std::optional& timeZoneName) { - return evaluateWithTimestampWithTimezone( + return evaluateWithTimestampWithTimezone( "date(c0)", timestamp, timeZoneName); }; // 1970-01-01 00:00:00.000 +00:00 - EXPECT_EQ(Date(), dateFunction(0, "+00:00")); - EXPECT_EQ(Date(), dateFunction(0, "Europe/London")); + EXPECT_EQ(0, dateFunction(0, "+00:00")); + EXPECT_EQ(0, dateFunction(0, "Europe/London")); // 1970-01-01 00:00:00.000 -08:00 - EXPECT_EQ(Date(-1), dateFunction(0, "-08:00")); - EXPECT_EQ(Date(-1), dateFunction(0, "America/Los_Angeles")); + EXPECT_EQ(-1, dateFunction(0, "-08:00")); + EXPECT_EQ(-1, dateFunction(0, "America/Los_Angeles")); // 1970-01-01 00:00:00.000 +08:00 - EXPECT_EQ(Date(), dateFunction(0, "+08:00")); - EXPECT_EQ(Date(), dateFunction(0, "Asia/Chongqing")); + EXPECT_EQ(0, dateFunction(0, "+08:00")); + EXPECT_EQ(0, dateFunction(0, "Asia/Chongqing")); // 1970-01-01 18:00:00.000 +08:00 - EXPECT_EQ(Date(1), dateFunction(18 * 3'600 * 1'000, "+08:00")); - EXPECT_EQ(Date(1), dateFunction(18 * 3'600 * 1'000, "Asia/Chongqing")); + EXPECT_EQ(1, dateFunction(18 * 3'600 * 1'000, "+08:00")); + EXPECT_EQ(1, dateFunction(18 * 3'600 * 1'000, "Asia/Chongqing")); // 1970-01-01 06:00:00.000 -08:00 - EXPECT_EQ(Date(-1), dateFunction(6 * 3'600 * 1'000, "-08:00")); - EXPECT_EQ(Date(-1), dateFunction(6 * 3'600 * 1'000, "America/Los_Angeles")); + EXPECT_EQ(-1, dateFunction(6 * 3'600 * 1'000, "-08:00")); + EXPECT_EQ(-1, dateFunction(6 * 3'600 * 1'000, "America/Los_Angeles")); // 2020-02-05 10:00:00.000 +08:00 EXPECT_EQ( - Date(18297), + 18297, dateFunction((18297 * kSecondsInDay + 10 * 3'600) * 1'000, "+08:00")); EXPECT_EQ( - Date(18297), + 18297, dateFunction( (18297 * kSecondsInDay + 10 * 3'600) * 1'000, "Asia/Chongqing")); // 2020-02-05 20:00:00.000 +08:00 EXPECT_EQ( - Date(18298), + 18298, dateFunction((18297 * kSecondsInDay + 20 * 3'600) * 1'000, "+08:00")); EXPECT_EQ( - Date(18298), + 18298, dateFunction( (18297 * kSecondsInDay + 20 * 3'600) * 1'000, "Asia/Chongqing")); // 2020-02-05 16:00:00.000 -08:00 EXPECT_EQ( - Date(18297), + 18297, dateFunction((18297 * kSecondsInDay + 16 * 3'600) * 1'000, "-08:00")); EXPECT_EQ( - Date(18297), + 18297, dateFunction( (18297 * kSecondsInDay + 16 * 3'600) * 1'000, "America/Los_Angeles")); // 2020-02-05 06:00:00.000 -08:00 EXPECT_EQ( - Date(18296), + 18296, dateFunction((18297 * kSecondsInDay + 6 * 3'600) * 1'000, "-08:00")); EXPECT_EQ( - Date(18296), + 18296, dateFunction( (18297 * kSecondsInDay + 6 * 3'600) * 1'000, "America/Los_Angeles")); // 1919-11-28 10:00:00.000 +08:00 EXPECT_EQ( - Date(-18297), + -18297, dateFunction((-18297 * kSecondsInDay + 10 * 3'600) * 1'000, "+08:00")); EXPECT_EQ( - Date(-18297), + -18297, dateFunction( (-18297 * kSecondsInDay + 10 * 3'600) * 1'000, "Asia/Chongqing")); // 1919-11-28 20:00:00.000 +08:00 EXPECT_EQ( - Date(-18296), + -18296, dateFunction((-18297 * kSecondsInDay + 20 * 3'600) * 1'000, "+08:00")); EXPECT_EQ( - Date(-18296), + -18296, dateFunction( (-18297 * kSecondsInDay + 20 * 3'600) * 1'000, "Asia/Chongqing")); // 1919-11-28 16:00:00.000 -08:00 EXPECT_EQ( - Date(-18297), + -18297, dateFunction((-18297 * kSecondsInDay + 16 * 3'600) * 1'000, "-08:00")); EXPECT_EQ( - Date(-18297), + -18297, dateFunction( (-18297 * kSecondsInDay + 16 * 3'600) * 1'000, "America/Los_Angeles")); // 1919-11-28 06:00:00.000 -08:00 EXPECT_EQ( - Date(-18298), + -18298, dateFunction((-18297 * kSecondsInDay + 6 * 3'600) * 1'000, "-08:00")); EXPECT_EQ( - Date(-18298), + -18298, dateFunction( (-18297 * kSecondsInDay + 6 * 3'600) * 1'000, "America/Los_Angeles")); } @@ -3087,13 +3091,13 @@ TEST_F(DateTimeFunctionsTest, currentDateWithTimezone) { auto tz = "America/Los_Angeles"; setQueryTimeZone(tz); auto dateBefore = getCurrentDate(tz); - auto result = evaluateOnce("current_date()", emptyRowVector); + auto result = evaluateOnce("current_date()", emptyRowVector); auto dateAfter = getCurrentDate(tz); EXPECT_TRUE(result.has_value()); EXPECT_LE(dateBefore, result); EXPECT_LE(result, dateAfter); - EXPECT_LE(dateAfter.days() - dateBefore.days(), 1); + EXPECT_LE(dateAfter - dateBefore, 1); } TEST_F(DateTimeFunctionsTest, currentDateWithoutTimezone) { @@ -3102,13 +3106,13 @@ TEST_F(DateTimeFunctionsTest, currentDateWithoutTimezone) { // Do not set the timezone, so the timezone obtained from QueryConfig // will be nullptr. auto dateBefore = getCurrentDate(std::nullopt); - auto result = evaluateOnce("current_date()", emptyRowVector); + auto result = evaluateOnce("current_date()", emptyRowVector); auto dateAfter = getCurrentDate(std::nullopt); EXPECT_TRUE(result.has_value()); EXPECT_LE(dateBefore, result); EXPECT_LE(result, dateAfter); - EXPECT_LE(dateAfter.days() - dateBefore.days(), 1); + EXPECT_LE(dateAfter - dateBefore, 1); } TEST_F(DateTimeFunctionsTest, timeZoneHour) { diff --git a/velox/functions/prestosql/tests/GreatestLeastTest.cpp b/velox/functions/prestosql/tests/GreatestLeastTest.cpp index 9eb5341dfbe..2176a81bf59 100644 --- a/velox/functions/prestosql/tests/GreatestLeastTest.cpp +++ b/velox/functions/prestosql/tests/GreatestLeastTest.cpp @@ -27,19 +27,22 @@ class GreatestLeastTest : public functions::test::FunctionBaseTest { const std::string& query, const std::vector>& inputs, const std::vector>& output, - std::optional stringBuffersExpectedCount = std::nullopt) { + std::optional stringBuffersExpectedCount = std::nullopt, + const TypePtr& type = CppToType::create(), + const TypePtr& resultType = CppToType::create()) { // Create input vectors auto vectorSize = inputs[0].size(); std::vector inputColumns(inputs.size()); for (auto i = 0; i < inputColumns.size(); ++i) { - inputColumns[i] = makeFlatVector(inputs[i]); + inputColumns[i] = makeFlatVector(inputs[i], type); for (auto j = 0; j < vectorSize; ++j) { inputColumns[i]->asFlatVector()->set(j, inputs[i][j]); } } // Call evaluate to run the query on the created input - auto result = evaluate>(query, makeRowVector(inputColumns)); + auto result = evaluate>( + query, makeRowVector(inputColumns), std::nullopt, resultType); for (int32_t i = 0; i < vectorSize; ++i) { if (output[i].has_value()) { ASSERT_EQ(result->valueAt(i), output[i]); @@ -200,21 +203,27 @@ TEST_F(GreatestLeastTest, leastTimeStamp) { } TEST_F(GreatestLeastTest, greatestDate) { - runTest( + runTest( "greatest(c0, c1, c2)", - {{Date(0), Date(5), Date(0)}, - {Date(1), Date(0), Date(-5)}, - {Date(5), Date(-5), Date(-10)}}, - {Date(5), Date(5), Date(0)}); + { + {0, 5, 0}, + {1, 0, -5}, + {5, -5, -10}, + }, + {5, 5, 0}, + std::nullopt, + DATE(), + DATE()); } TEST_F(GreatestLeastTest, leastDate) { - runTest( + runTest( "least(c0, c1, c2)", - {{Date(0), Date(0), Date(5)}, - {Date(1), Date(-1), Date(-1)}, - {Date(5), Date(5), Date(-5)}}, - {Date(0), Date(-1), Date(-5)}); + {{0, 0, 5}, {1, -1, -1}, {5, 5, -5}}, + {0, -1, -5}, + std::nullopt, + DATE(), + DATE()); } TEST_F(GreatestLeastTest, stringBuffersMoved) { diff --git a/velox/functions/prestosql/tests/InPredicateTest.cpp b/velox/functions/prestosql/tests/InPredicateTest.cpp index 24b0ab8dfc0..540d2754cb4 100644 --- a/velox/functions/prestosql/tests/InPredicateTest.cpp +++ b/velox/functions/prestosql/tests/InPredicateTest.cpp @@ -319,11 +319,10 @@ TEST_F(InPredicateTest, varbinary) { } TEST_F(InPredicateTest, date) { - auto dateValue = Date(); - parseTo("2000-01-01", dateValue); + auto dateValue = DATE()->toDays("2000-01-01"); auto input = makeRowVector({ - makeNullableFlatVector({dateValue}, DATE()), + makeNullableFlatVector({dateValue}, DATE()), }); assertEqualVectors( diff --git a/velox/functions/prestosql/tests/JsonCastTest.cpp b/velox/functions/prestosql/tests/JsonCastTest.cpp index 93940ca5750..acf6ef0a894 100644 --- a/velox/functions/prestosql/tests/JsonCastTest.cpp +++ b/velox/functions/prestosql/tests/JsonCastTest.cpp @@ -288,12 +288,12 @@ TEST_F(JsonCastTest, fromDouble) { } TEST_F(JsonCastTest, fromDate) { - testCast( + testCast( DATE(), JSON(), {0, 1000, -10000, std::nullopt}, {"1970-01-01"_sv, "1972-09-27"_sv, "1942-08-16"_sv, std::nullopt}); - testCast( + testCast( DATE(), JSON(), {std::nullopt, std::nullopt, std::nullopt, std::nullopt}, @@ -1070,7 +1070,7 @@ TEST_F(JsonCastTest, toArrayAndMapOfJson) { TEST_F(JsonCastTest, toInvalid) { testThrow( JSON(), TIMESTAMP(), {"null"_sv}, "Cannot cast JSON to TIMESTAMP"); - testThrow( + testThrow( JSON(), DATE(), {"null"_sv}, "Cannot cast JSON to DATE"); // Casting JSON arrays to ROW type with different number of fields or diff --git a/velox/functions/prestosql/tests/SequenceTest.cpp b/velox/functions/prestosql/tests/SequenceTest.cpp index 4db08a87ed1..1b19da3ec91 100644 --- a/velox/functions/prestosql/tests/SequenceTest.cpp +++ b/velox/functions/prestosql/tests/SequenceTest.cpp @@ -44,10 +44,8 @@ class SequenceTest : public FunctionBaseTest { }; } // namespace -Date parseDate(const std::string& dateStr) { - Date returnDate; - parseTo(dateStr, returnDate); - return returnDate; +int32_t parseDate(const std::string& dateStr) { + return DATE()->toDays(dateStr); } TEST_F(SequenceTest, sequence) { @@ -137,59 +135,48 @@ TEST_F(SequenceTest, invalidStep) { } TEST_F(SequenceTest, dateArguments) { - const auto startVector = - makeFlatVector({Date(1991), Date(1992), Date(1992)}); - const auto stopVector = - makeFlatVector({Date(1996), Date(1988), Date(1992)}); - const auto expected = makeArrayVector( - {{Date(1991), Date(1992), Date(1993), Date(1994), Date(1995), Date(1996)}, - {Date(1992), Date(1991), Date(1990), Date(1989), Date(1988)}, - {Date(1992)}}); + const auto startVector = makeFlatVector({1991, 1992, 1992}, DATE()); + const auto stopVector = makeFlatVector({1996, 1988, 1992}, DATE()); + const auto expected = makeArrayVector( + {{1991, 1992, 1993, 1994, 1995, 1996}, + {1992, 1991, 1990, 1989, 1988}, + {1992}}, + DATE()); testExpression("sequence(C0, C1)", {startVector, stopVector}, expected); } TEST_F(SequenceTest, dateArgumentsExceedMaxEntries) { - const auto startVector = - makeFlatVector({Date(1991), Date(1992), Date(1992)}); - const auto stopVector = - makeFlatVector({Date(1996), Date(198800), Date(1992)}); + const auto startVector = makeFlatVector({1991, 1992, 1992}, DATE()); + const auto stopVector = makeFlatVector({1996, 198800, 1992}, DATE()); testExpressionWithError( "sequence(C0, C1)", {startVector, stopVector}, "result of sequence function must not have more than 10000 entries"); - auto expected = makeNullableArrayVector( - {{{Date(1991), - Date(1992), - Date(1993), - Date(1994), - Date(1995), - Date(1996)}}, - std::nullopt, - {{Date(1992)}}}); + auto expected = makeNullableArrayVector( + {{{1991, 1992, 1993, 1994, 1995, 1996}}, std::nullopt, {{1992}}}, + ARRAY(DATE())); testExpression("try(sequence(C0, C1))", {startVector, stopVector}, expected); } TEST_F(SequenceTest, dateIntervalDayStep) { int64_t day = 86400000; // 24 * 60 * 60 * 1000 - const auto startVector = makeFlatVector({Date(1991), Date(1992)}); - const auto stopVector = makeFlatVector({Date(1996), Date(2000)}); + const auto startVector = makeFlatVector({1991, 1992}, DATE()); + const auto stopVector = makeFlatVector({1996, 2000}, DATE()); const auto stepVector = makeFlatVector({day, 2 * day}, INTERVAL_DAY_TIME()); - const auto expected = makeArrayVector( - {{Date(1991), Date(1992), Date(1993), Date(1994), Date(1995), Date(1996)}, - {Date(1992), Date(1994), Date(1996), Date(1998), Date(2000)}}); + const auto expected = makeArrayVector( + {{1991, 1992, 1993, 1994, 1995, 1996}, {1992, 1994, 1996, 1998, 2000}}, + DATE()); testExpression( "sequence(C0, C1, C2)", {startVector, stopVector, stepVector}, expected); } TEST_F(SequenceTest, dateInvalidIntervalDayStep) { int64_t day = 86400000; // 24 * 60 * 60 * 1000 - const auto startVector = - makeFlatVector({Date(1991), Date(1992), Date(1992)}); - const auto stopVector = - makeFlatVector({Date(1996), Date(2000), Date(2000)}); + const auto startVector = makeFlatVector({1991, 1992, 1992}, DATE()); + const auto stopVector = makeFlatVector({1996, 2000, 2000}, DATE()); auto stepVector = makeFlatVector({-1 * day, 0, 1}, INTERVAL_DAY_TIME()); testExpressionWithError( @@ -212,8 +199,8 @@ TEST_F(SequenceTest, dateInvalidIntervalDayStep) { "sequence step must be a day interval if start and end values are dates"); stepVector = makeFlatVector({1, -1 * day, 0}, INTERVAL_DAY_TIME()); - auto expected = - makeNullableArrayVector({std::nullopt, std::nullopt, std::nullopt}); + auto expected = makeNullableArrayVector( + {std::nullopt, std::nullopt, std::nullopt}, ARRAY(DATE())); testExpression( "try(sequence(C0, C1, C2))", {startVector, stopVector, stepVector}, @@ -221,43 +208,46 @@ TEST_F(SequenceTest, dateInvalidIntervalDayStep) { } TEST_F(SequenceTest, dateYearMonthStep) { - const auto startVector = makeFlatVector( + const auto startVector = makeFlatVector( {parseDate("1975-01-31"), parseDate("1975-03-15"), - parseDate("2023-12-31")}); - const auto stopVector = makeFlatVector( + parseDate("2023-12-31")}, + DATE()); + const auto stopVector = makeFlatVector( {parseDate("1975-06-20"), parseDate("1974-12-15"), - parseDate("2024-05-31")}); + parseDate("2024-05-31")}, + DATE()); const auto stepVector = makeFlatVector({1, -1, 2}, INTERVAL_YEAR_MONTH()); - const auto expected = - makeArrayVector({// last day of Feb - // result won't include 1975-06-20 - {parseDate("1975-01-31"), - parseDate("1975-02-28"), - parseDate("1975-03-31"), - parseDate("1975-04-30"), - parseDate("1975-05-31")}, - // negative step - {parseDate("1975-03-15"), - parseDate("1975-02-15"), - parseDate("1975-01-15"), - parseDate("1974-12-15")}, - // leap year - {parseDate("2023-12-31"), - parseDate("2024-02-29"), - parseDate("2024-04-30")}}); + const auto expected = makeArrayVector( + {// last day of Feb + // result won't include 1975-06-20 + {parseDate("1975-01-31"), + parseDate("1975-02-28"), + parseDate("1975-03-31"), + parseDate("1975-04-30"), + parseDate("1975-05-31")}, + // negative step + {parseDate("1975-03-15"), + parseDate("1975-02-15"), + parseDate("1975-01-15"), + parseDate("1974-12-15")}, + // leap year + {parseDate("2023-12-31"), + parseDate("2024-02-29"), + parseDate("2024-04-30")}}, + DATE()); testExpression( "sequence(C0, C1, C2)", {startVector, stopVector, stepVector}, expected); } TEST_F(SequenceTest, dateInvalidYearMonthStep) { - const auto startVector = - makeFlatVector({parseDate("1975-01-31"), parseDate("1975-03-15")}); - const auto stopVector = - makeFlatVector({parseDate("1975-06-01"), parseDate("1974-12-15")}); + const auto startVector = makeFlatVector( + {parseDate("1975-01-31"), parseDate("1975-03-15")}, DATE()); + const auto stopVector = makeFlatVector( + {parseDate("1975-06-01"), parseDate("1974-12-15")}, DATE()); auto stepVector = makeFlatVector({0, 0}, INTERVAL_DAY_TIME()); testExpressionWithError( @@ -273,13 +263,14 @@ TEST_F(SequenceTest, dateInvalidYearMonthStep) { "step is greater than zero otherwise stop should be less than or equal to " "start"); - auto expected = makeNullableArrayVector( + auto expected = makeNullableArrayVector( {{{parseDate("1975-01-31"), parseDate("1975-02-28"), parseDate("1975-03-31"), parseDate("1975-04-30"), parseDate("1975-05-31")}}, - std::nullopt}); + std::nullopt}, + ARRAY(DATE())); testExpression( "try(sequence(C0, C1, C2))", {startVector, stopVector, stepVector}, @@ -287,10 +278,10 @@ TEST_F(SequenceTest, dateInvalidYearMonthStep) { } TEST_F(SequenceTest, dateIntervalExceedMaxEntries) { - const auto startVector = - makeFlatVector({parseDate("1975-01-31"), parseDate("1975-03-15")}); - const auto stopVector = - makeFlatVector({parseDate("3975-06-01"), parseDate("3974-12-15")}); + const auto startVector = makeFlatVector( + {parseDate("1975-01-31"), parseDate("1975-03-15")}, DATE()); + const auto stopVector = makeFlatVector( + {parseDate("3975-06-01"), parseDate("3974-12-15")}, DATE()); auto stepVector = makeFlatVector({1, 1}, INTERVAL_YEAR_MONTH()); testExpressionWithError( diff --git a/velox/functions/prestosql/tests/utils/FunctionBaseTest.h b/velox/functions/prestosql/tests/utils/FunctionBaseTest.h index 0e4fd06e073..65654f72d28 100644 --- a/velox/functions/prestosql/tests/utils/FunctionBaseTest.h +++ b/velox/functions/prestosql/tests/utils/FunctionBaseTest.h @@ -104,29 +104,32 @@ class FunctionBaseTest : public testing::Test, std::shared_ptr evaluate( const core::TypedExprPtr& typedExpr, const RowVectorPtr& data, - const std::optional& rows = std::nullopt) { + const std::optional& rows = std::nullopt, + const TypePtr& resultType = nullptr) { auto result = evaluate(typedExpr, data, rows); - return castEvaluateResult(result, typedExpr->toString()); + return castEvaluateResult(result, typedExpr->toString(), resultType); } template std::shared_ptr evaluate( const std::string& expression, const RowVectorPtr& data, - const std::optional& rows = std::nullopt) { + const std::optional& rows = std::nullopt, + const TypePtr& resultType = nullptr) { auto result = evaluate(expression, data, rows); - return castEvaluateResult(result, expression); + return castEvaluateResult(result, expression, resultType); } template std::shared_ptr evaluateSimplified( const std::string& expression, const RowVectorPtr& data, - const std::optional& rows = std::nullopt) { + const std::optional& rows = std::nullopt, + const TypePtr& resultType = nullptr) { auto typedExpr = makeTypedExpr(expression, asRowType(data->type())); auto result = evaluateImpl(typedExpr, data, rows); - return castEvaluateResult(result, expression); + return castEvaluateResult(result, expression, resultType); } template @@ -134,7 +137,8 @@ class FunctionBaseTest : public testing::Test, const std::string& expression, RowVectorPtr data, const SelectivityVector& rows, - VectorPtr& result) { + VectorPtr& result, + const TypePtr& resultType = nullptr) { exec::ExprSet exprSet( {makeTypedExpr(expression, asRowType(data->type()))}, &execCtx_); @@ -143,6 +147,11 @@ class FunctionBaseTest : public testing::Test, exprSet.eval(rows, evalCtx, results); result = results[0]; + // reinterpret_cast is used for functions that return logical types like + // DATE(). + if (resultType != nullptr) { + return std::reinterpret_pointer_cast(results[0]); + } return std::dynamic_pointer_cast(results[0]); } @@ -171,23 +180,25 @@ class FunctionBaseTest : public testing::Test, const std::string& expr, const std::vector>& args, const std::vector& types, - const std::optional& rows = std::nullopt) { + const std::optional& rows = std::nullopt, + const TypePtr& resultType = nullptr) { std::vector flatVectors; for (vector_size_t i = 0; i < args.size(); ++i) { flatVectors.emplace_back(makeNullableFlatVector( std::vector>{args[i]}, types[i])); } auto rowVectorPtr = makeRowVector(flatVectors); - return evaluateOnce(expr, rowVectorPtr, rows); + return evaluateOnce(expr, rowVectorPtr, rows, resultType); } template std::optional evaluateOnce( const std::string& expr, const RowVectorPtr rowVectorPtr, - const std::optional& rows = std::nullopt) { - auto result = - evaluate>>(expr, rowVectorPtr, rows); + const std::optional& rows = std::nullopt, + const TypePtr& resultType = nullptr) { + auto result = evaluate>>( + expr, rowVectorPtr, rows, resultType); return result->isNullAt(0) ? std::optional{} : ReturnType(result->valueAt(0)); } @@ -247,8 +258,22 @@ class FunctionBaseTest : public testing::Test, template std::shared_ptr castEvaluateResult( const VectorPtr& result, - const std::string& expression) { + const std::string& expression, + const TypePtr& expectedType = nullptr) { + // reinterpret_cast is used for functions that return logical types + // like DATE(). VELOX_CHECK(result, "Expression evaluation result is null: {}", expression); + if (expectedType != nullptr) { + VELOX_CHECK_EQ( + result->type()->kind(), + expectedType->kind(), + "Expression evaluation result is not of expected type kind: {} -> {} vector of type {}", + expression, + result->encoding(), + result->type()->kindName()); + auto castedResult = std::reinterpret_pointer_cast(result); + return castedResult; + } auto castedResult = std::dynamic_pointer_cast(result); VELOX_CHECK( diff --git a/velox/functions/prestosql/types/JsonType.cpp b/velox/functions/prestosql/types/JsonType.cpp index 3ce58a8a29c..33ee34529b1 100644 --- a/velox/functions/prestosql/types/JsonType.cpp +++ b/velox/functions/prestosql/types/JsonType.cpp @@ -39,7 +39,8 @@ template void generateJsonTyped( const SimpleVector& input, int row, - std::string& result) { + std::string& result, + const TypePtr& type) { auto value = input.valueAt(row); if constexpr (std::is_same_v) { @@ -54,9 +55,10 @@ void generateJsonTyped( if constexpr (std::is_same_v) { result.append(value ? "true" : "false"); - } else if constexpr ( - std::is_same_v || std::is_same_v) { + } else if constexpr (std::is_same_v) { result.append(std::to_string(value)); + } else if (type->isDate()) { + result.append(DATE()->toString(value)); } else { folly::toAppend(value, &result); } @@ -89,7 +91,7 @@ void castToJson( flatResult.set(row, "null"); } else { result.clear(); - generateJsonTyped(*inputVector, row, result); + generateJsonTyped(*inputVector, row, result, input.type()); flatResult.set(row, StringView{result}); } @@ -100,7 +102,7 @@ void castToJson( VELOX_FAIL("Map keys cannot be null."); } else { result.clear(); - generateJsonTyped(*inputVector, row, result); + generateJsonTyped(*inputVector, row, result, input.type()); flatResult.set(row, StringView{result}); } @@ -757,7 +759,6 @@ bool JsonCastOperator::isSupportedFromType(const TypePtr& other) const { switch (other->kind()) { case TypeKind::UNKNOWN: - case TypeKind::DATE: case TypeKind::TIMESTAMP: return true; case TypeKind::ARRAY: @@ -779,6 +780,10 @@ bool JsonCastOperator::isSupportedFromType(const TypePtr& other) const { } bool JsonCastOperator::isSupportedToType(const TypePtr& other) const { + if (other->isDate()) { + return false; + } + if (isSupportedBasicType(other)) { return true; } diff --git a/velox/functions/sparksql/DateTimeFunctions.h b/velox/functions/sparksql/DateTimeFunctions.h index ee1a830efc6..277814697f7 100644 --- a/velox/functions/sparksql/DateTimeFunctions.h +++ b/velox/functions/sparksql/DateTimeFunctions.h @@ -169,7 +169,7 @@ struct MakeDateFunction { year, month, day); - result = Date(daysSinceEpoch); + result = daysSinceEpoch; } }; @@ -205,7 +205,7 @@ struct LastDayFunction { year, month, day); - result = Date(daysSinceEpoch); + result = daysSinceEpoch; } }; diff --git a/velox/functions/sparksql/LeastGreatest.cpp b/velox/functions/sparksql/LeastGreatest.cpp index f054c74c673..1f5750698d1 100644 --- a/velox/functions/sparksql/LeastGreatest.cpp +++ b/velox/functions/sparksql/LeastGreatest.cpp @@ -118,7 +118,6 @@ std::shared_ptr makeImpl( SCALAR_CASE(VARCHAR) SCALAR_CASE(VARBINARY) SCALAR_CASE(TIMESTAMP) - SCALAR_CASE(DATE) #undef SCALAR_CASE default: VELOX_NYI( diff --git a/velox/functions/sparksql/aggregates/FirstLastAggregate.cpp b/velox/functions/sparksql/aggregates/FirstLastAggregate.cpp index 73e228c6332..00c4dbb4741 100644 --- a/velox/functions/sparksql/aggregates/FirstLastAggregate.cpp +++ b/velox/functions/sparksql/aggregates/FirstLastAggregate.cpp @@ -312,8 +312,6 @@ exec::AggregateRegistrationResult registerFirstLast(const std::string& name) { case TypeKind::TIMESTAMP: return std::make_unique>( resultType); - case TypeKind::DATE: - return std::make_unique>(resultType); case TypeKind::VARCHAR: case TypeKind::ARRAY: case TypeKind::MAP: diff --git a/velox/functions/sparksql/aggregates/tests/FirstAggregateTest.cpp b/velox/functions/sparksql/aggregates/tests/FirstAggregateTest.cpp index 0a001f3912b..36edf289b27 100644 --- a/velox/functions/sparksql/aggregates/tests/FirstAggregateTest.cpp +++ b/velox/functions/sparksql/aggregates/tests/FirstAggregateTest.cpp @@ -197,29 +197,30 @@ TEST_F(FirstAggregateTest, timestampGlobal) { TEST_F(FirstAggregateTest, dateGroupBy) { auto vectors = {makeRowVector({ makeFlatVector(98, [](auto row) { return row % 7; }), - makeFlatVector( + makeFlatVector( 98, // size - [](auto row) { return Date(row); }, // valueAt - nullEvery(3)), + [](auto row) { return row; }, // valueAt + nullEvery(3), + DATE()), })}; auto ignoreNullData = {makeRowVector({ makeFlatVector(7, [](auto row) { return row; }), - makeFlatVector( + makeFlatVector( 7, // size - [](auto row) { - return row % 3 == 0 ? Date(row + 7) : Date(row); - } // valueAt - ), + [](auto row) { return row % 3 == 0 ? row + 7 : row; }, // valueAt + nullptr, + DATE()), })}; // Expected result should have first 7 rows including nulls. auto hasNullData = {makeRowVector({ makeFlatVector(7, [](auto row) { return row; }), - makeFlatVector( + makeFlatVector( 7, // size - [](auto row) { return Date(row); }, // valueAt - [](auto row) { return row % 3 == 0; }), // nullAt + [](auto row) { return row; }, // valueAt + [](auto row) { return row % 3 == 0; }, + DATE()), // nullAt })}; testGroupBy(vectors, ignoreNullData, hasNullData); @@ -227,14 +228,15 @@ TEST_F(FirstAggregateTest, dateGroupBy) { TEST_F(FirstAggregateTest, dateGlobal) { auto vectors = {makeRowVector({ - makeNullableFlatVector( - {std::nullopt, Date(1), Date(2), std::nullopt}), + makeNullableFlatVector( + {std::nullopt, 1, 2, std::nullopt}, DATE()), })}; - auto ignoreNullData = {makeRowVector({makeFlatVector({Date(1)})})}; + auto ignoreNullData = { + makeRowVector({makeNullableFlatVector({1}, DATE())})}; auto hasNullData = { - makeRowVector({makeNullableFlatVector({std::nullopt})})}; + makeRowVector({makeNullableFlatVector({std::nullopt}, DATE())})}; testGlobalAggregate(vectors, ignoreNullData, hasNullData); } diff --git a/velox/functions/sparksql/aggregates/tests/LastAggregateTest.cpp b/velox/functions/sparksql/aggregates/tests/LastAggregateTest.cpp index 42676bd739e..45daf8e1af5 100644 --- a/velox/functions/sparksql/aggregates/tests/LastAggregateTest.cpp +++ b/velox/functions/sparksql/aggregates/tests/LastAggregateTest.cpp @@ -195,29 +195,32 @@ TEST_F(LastAggregateTest, timestampGlobal) { TEST_F(LastAggregateTest, dateGroupBy) { auto vectors = {makeRowVector({ makeFlatVector(98, [](auto row) { return row % 7; }), - makeFlatVector( + makeFlatVector( 98, // size - [](auto row) { return Date(row); }, // valueAt - [](auto row) { return row % 3 == 0; }), // nullAt + [](auto row) { return row; }, // valueAt + [](auto row) { return row % 3 == 0; }, // nullAt + DATE()), })}; auto ignoreNullData = {makeRowVector({ makeFlatVector(7, [](auto row) { return row; }), - makeFlatVector( + makeFlatVector( 7, // size [](auto row) { - return (row + 91) % 3 == 0 ? Date(row + 91 - 7) : Date(row + 91); - } // valueAt - ), + return (row + 91) % 3 == 0 ? row + 91 - 7 : row + 91; + }, // valueAt, + nullptr, // nullAt + DATE()), })}; // Expected result should have first 7 rows including nulls. auto hasNullData = {makeRowVector({ makeFlatVector(7, [](auto row) { return row; }), - makeFlatVector( + makeFlatVector( 7, // size - [](auto row) { return Date(row + 91); }, // valueAt - [](auto row) { return (row + 91) % 3 == 0; }), // nullAt + [](auto row) { return row + 91; }, // valueAt + [](auto row) { return (row + 91) % 3 == 0; }, // nullAt + DATE()), })}; testGroupBy(vectors, ignoreNullData, hasNullData); @@ -225,15 +228,15 @@ TEST_F(LastAggregateTest, dateGroupBy) { TEST_F(LastAggregateTest, dateGlobal) { auto vectors = {makeRowVector({ - makeNullableFlatVector( - {std::nullopt, Date(1), Date(2), std::nullopt}), + makeNullableFlatVector( + {std::nullopt, 1, 2, std::nullopt}, DATE()), })}; auto ignoreNullData = { - makeRowVector({makeNullableFlatVector({Date(2)})})}; + makeRowVector({makeNullableFlatVector({2}, DATE())})}; auto hasNullData = { - makeRowVector({makeNullableFlatVector({std::nullopt})})}; + makeRowVector({makeNullableFlatVector({std::nullopt}, DATE())})}; testGlobalAggregate(vectors, ignoreNullData, hasNullData); } diff --git a/velox/functions/sparksql/tests/ArraySortTest.cpp b/velox/functions/sparksql/tests/ArraySortTest.cpp index 845cb9ceaa2..4f9d5195563 100644 --- a/velox/functions/sparksql/tests/ArraySortTest.cpp +++ b/velox/functions/sparksql/tests/ArraySortTest.cpp @@ -88,8 +88,8 @@ TEST_F(ArraySortTest, timestamp) { } TEST_F(ArraySortTest, date) { - auto input = makeNullableArrayVector(dateInput()); - auto expected = makeNullableArrayVector(dateAscNullLargest()); + auto input = makeNullableArrayVector(dateInput(), ARRAY(DATE())); + auto expected = makeNullableArrayVector(dateAscNullLargest(), ARRAY(DATE())); testArraySort(input, expected); } diff --git a/velox/functions/sparksql/tests/ArraySortTestData.h b/velox/functions/sparksql/tests/ArraySortTestData.h index 9f2c443f919..b7e44c0f2e0 100644 --- a/velox/functions/sparksql/tests/ArraySortTestData.h +++ b/velox/functions/sparksql/tests/ArraySortTestData.h @@ -182,30 +182,27 @@ inline NestedVector> timestampAscNullLargest() { }; } -inline NestedVector> dateInput() { - using D = Date; - return NestedVector>{ +inline NestedVector> dateInput() { + return NestedVector>{ {}, {std::nullopt, std::nullopt}, - {D{0}, D{1}, std::nullopt, D{4}, D{3}}, + {0, 1, std::nullopt, 4, 3}, }; } -inline NestedVector> dateAscNullSmallest() { - using D = Date; - return NestedVector>{ +inline NestedVector> dateAscNullSmallest() { + return NestedVector>{ {}, {std::nullopt, std::nullopt}, - {std::nullopt, D{0}, D{1}, D{3}, D{4}}, + {std::nullopt, 0, 1, 3, 4}, }; } -inline NestedVector> dateAscNullLargest() { - using D = Date; - return NestedVector>{ +inline NestedVector> dateAscNullLargest() { + return NestedVector>{ {}, {std::nullopt, std::nullopt}, - {D{0}, D{1}, D{3}, D{4}, std::nullopt}, + {0, 1, 3, 4, std::nullopt}, }; } diff --git a/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp b/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp index 87728376591..88dae6a02da 100644 --- a/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp +++ b/velox/functions/sparksql/tests/DateTimeFunctionsTest.cpp @@ -56,14 +56,14 @@ TEST_F(DateTimeFunctionsTest, year) { } TEST_F(DateTimeFunctionsTest, yearDate) { - const auto year = [&](std::optional date) { - return evaluateOnce("year(c0)", date); + const auto year = [&](std::optional date) { + return evaluateOnce("year(c0)", {date}, {DATE()}); }; EXPECT_EQ(std::nullopt, year(std::nullopt)); - EXPECT_EQ(1970, year(Date(0))); - EXPECT_EQ(1969, year(Date(-1))); - EXPECT_EQ(2020, year(Date(18262))); - EXPECT_EQ(1920, year(Date(-18262))); + EXPECT_EQ(1970, year(DATE()->toDays("1970-05-05"))); + EXPECT_EQ(1969, year(DATE()->toDays("1969-12-31"))); + EXPECT_EQ(2020, year(DATE()->toDays("2020-01-01"))); + EXPECT_EQ(1920, year(DATE()->toDays("1920-01-01"))); } TEST_F(DateTimeFunctionsTest, unixTimestamp) { @@ -147,14 +147,10 @@ TEST_F(DateTimeFunctionsTest, makeDate) { const auto makeDate = [&](std::optional year, std::optional month, std::optional day) { - return evaluateOnce("make_date(c0, c1, c2)", year, month, day); + return evaluateOnce("make_date(c0, c1, c2)", year, month, day); }; - Date expectedDate; - parseTo("1920-01-25", expectedDate); - EXPECT_EQ(makeDate(1920, 1, 25), expectedDate); - - parseTo("-0010-01-30", expectedDate); - EXPECT_EQ(makeDate(-10, 1, 30), expectedDate); + EXPECT_EQ(makeDate(1920, 1, 25), DATE()->toDays("1920-01-25")); + EXPECT_EQ(makeDate(-10, 1, 30), DATE()->toDays("-0010-01-30")); constexpr int32_t kMax = std::numeric_limits::max(); auto errorMessage = fmt::format("Date out of range: {}-12-15", kMax); @@ -167,29 +163,23 @@ TEST_F(DateTimeFunctionsTest, makeDate) { VELOX_ASSERT_THROW(makeDate(2022, 3, 35), "Date out of range: 2022-3-35"); VELOX_ASSERT_THROW(makeDate(2023, 4, 31), "Date out of range: 2023-4-31"); - parseTo("2023-03-31", expectedDate); - EXPECT_EQ(makeDate(2023, 3, 31), expectedDate); + EXPECT_EQ(makeDate(2023, 3, 31), DATE()->toDays("2023-03-31")); VELOX_ASSERT_THROW(makeDate(2023, 2, 29), "Date out of range: 2023-2-29"); - parseTo("2023-03-29", expectedDate); - EXPECT_EQ(makeDate(2023, 3, 29), expectedDate); + EXPECT_EQ(makeDate(2023, 3, 29), DATE()->toDays("2023-03-29")); } TEST_F(DateTimeFunctionsTest, lastDay) { - const auto lastDayFunc = [&](const std::optional date) { - return evaluateOnce("last_day(c0)", date); + const auto lastDayFunc = [&](const std::optional date) { + return evaluateOnce("last_day(c0)", {date}, {DATE()}); }; const auto lastDay = [&](const std::string& dateStr) { - Date d0; - parseTo(dateStr, d0); - return lastDayFunc(d0); + return lastDayFunc(DATE()->toDays(dateStr)); }; const auto parseDateStr = [&](const std::string& dateStr) { - Date d0; - parseTo(dateStr, d0); - return d0; + return DATE()->toDays(dateStr); }; EXPECT_EQ(lastDay("2015-02-28"), parseDateStr("2015-02-28")); diff --git a/velox/functions/sparksql/tests/InTest.cpp b/velox/functions/sparksql/tests/InTest.cpp index 6855ba1ec76..7738226707e 100644 --- a/velox/functions/sparksql/tests/InTest.cpp +++ b/velox/functions/sparksql/tests/InTest.cpp @@ -67,17 +67,18 @@ class InTest : public SparkFunctionBaseTest { } template - auto in(std::optional lhs, std::vector> rhs) { + auto in( + std::optional lhs, + std::vector> rhs, + const TypePtr& argType = CppToType::create()) { // We don't use evaluateOnce() because we can't get NaN through the DuckDB // parser. - auto getExpr = [&](bool asDictionary) { std::vector> args; - const auto argType = CppToType::create(); args.push_back( std::make_shared(argType, "c0")); - VectorPtr rhsArrayVector = - vectorMaker_.arrayVectorNullable({std::optional(rhs)}); + VectorPtr rhsArrayVector = vectorMaker_.arrayVectorNullable( + {std::optional(rhs)}, ARRAY(argType)); if (asDictionary) { auto indices = makeIndices(rhs.size(), [](auto /*row*/) { return 0; }); rhsArrayVector = wrapInDictionary(indices, rhs.size(), rhsArrayVector); @@ -104,9 +105,9 @@ class InTest : public SparkFunctionBaseTest { auto testForDictionary = [&](bool asDictionary = false) { auto lhsVector = - makeRowVector({makeNullableFlatVector(std::vector{lhs})}); + makeRowVector({makeNullableFlatVector(std::vector{lhs}, argType)}); auto flatResult = eval(lhsVector, asDictionary); - auto lhsConstantVector = makeRowVector({makeConstant(lhs, 1)}); + auto lhsConstantVector = makeRowVector({makeConstant(lhs, 1, argType)}); auto constResult = eval(lhsConstantVector, asDictionary); CHECK(flatResult == constResult) << "flatResult=" @@ -179,8 +180,8 @@ TEST_F(InTest, Timestamp) { } TEST_F(InTest, Date) { - EXPECT_EQ(in(Date(0), {Date(1), std::nullopt}), std::nullopt); - EXPECT_EQ(in(Date(0), {Date(0), Date()}), true); + EXPECT_EQ(in(0, {1, std::nullopt}, DATE()), std::nullopt); + EXPECT_EQ(in(0, {0}, DATE()), true); } TEST_F(InTest, Bool) { diff --git a/velox/functions/sparksql/tests/LeastGreatestTest.cpp b/velox/functions/sparksql/tests/LeastGreatestTest.cpp index 1791259f4f7..729b8f767be 100644 --- a/velox/functions/sparksql/tests/LeastGreatestTest.cpp +++ b/velox/functions/sparksql/tests/LeastGreatestTest.cpp @@ -22,9 +22,13 @@ namespace { class LeastTest : public SparkFunctionBaseTest { protected: template - std::optional - least(std::optional arg0, std::optional arg1, std::optional arg2) { - return evaluateOnce("least(c0, c1, c2)", arg0, arg1, arg2); + std::optional least( + std::optional arg0, + std::optional arg1, + std::optional arg2, + const TypePtr& type = CppToType::create()) { + return evaluateOnce( + "least(c0, c1, c2)", {arg0, arg1, arg2}, {type, type, type}); } template @@ -32,27 +36,35 @@ class LeastTest : public SparkFunctionBaseTest { std::optional arg0, std::optional arg1, std::optional arg2, - std::optional arg3) { - return evaluateOnce("least(c0, c1, c2, c3)", arg0, arg1, arg2, arg3); + std::optional arg3, + const TypePtr& type = CppToType::create()) { + return evaluateOnce( + "least(c0, c1, c2, c3)", + {arg0, arg1, arg2, arg3}, + {type, type, type, type}); } template - void flat() { + void flat(const TypePtr& type = CppToType::create()) { vector_size_t size = 20; // {0, null, null, 3, null, null, 6, null, null, ...}. auto first = makeFlatVector( size, [](vector_size_t row) { return row; }, - [](vector_size_t row) { return row % 3 != 0; }); + [](vector_size_t row) { return row % 3 != 0; }, + type); // {0, 10, null, 30, 40, null, 60, 70, null, ...}. auto second = makeFlatVector( - size, [](vector_size_t row) { return row * 10; }, nullEvery(3, 2)); + size, + [](vector_size_t row) { return row * 10; }, + nullEvery(3, 2), + type); // {0, 100, 200, 300, 400, 500, 600, 700, 800, ...}. - auto third = - makeFlatVector(size, [](vector_size_t row) { return row * 100; }); + auto third = makeFlatVector( + size, [](vector_size_t row) { return row * 100; }, nullptr, type); auto data = makeRowVector({first, second, third}); @@ -81,17 +93,18 @@ class LeastTest : public SparkFunctionBaseTest { } template - void constant() { + void constant(const TypePtr& type = CppToType::create()) { vector_size_t size = 20; // {0, null, null, 3, null, null, 6, null, null, ...}. auto first = makeFlatVector( size, [](vector_size_t row) { return row; }, - [](vector_size_t row) { return row % 3 != 0; }); + [](vector_size_t row) { return row % 3 != 0; }, + type); // {9, 9, 9, ...}. - auto second = makeConstant(9, size); + auto second = makeConstant(9, size, type); auto data = makeRowVector({first, second}); @@ -116,6 +129,9 @@ TEST_F(LeastTest, integral) { flat(); constant(); + + flat(DATE()); + constant(DATE()); } TEST_F(LeastTest, floating) { @@ -157,7 +173,7 @@ TEST_F(LeastTest, timestamp) { } TEST_F(LeastTest, date) { - EXPECT_EQ(least(Date(100), Date(1000), Date(10000)), Date(100)); + EXPECT_EQ(least(100, 1000, 10000, DATE()), 100); } class GreatestTest : public SparkFunctionBaseTest { @@ -166,8 +182,10 @@ class GreatestTest : public SparkFunctionBaseTest { std::optional greatest( std::optional arg0, std::optional arg1, - std::optional arg2) { - return evaluateOnce("greatest(c0, c1, c2)", arg0, arg1, arg2); + std::optional arg2, + const TypePtr& type = CppToType::create()) { + return evaluateOnce( + "greatest(c0, c1, c2)", {arg0, arg1, arg2}, {type, type, type}); } template @@ -175,26 +193,35 @@ class GreatestTest : public SparkFunctionBaseTest { std::optional arg0, std::optional arg1, std::optional arg2, - std::optional arg3) { - return evaluateOnce("greatest(c0, c1, c2, c3)", arg0, arg1, arg2, arg3); + std::optional arg3, + const TypePtr& type = CppToType::create()) { + return evaluateOnce( + "greatest(c0, c1, c2, c3)", + {arg0, arg1, arg2, arg3}, + {type, type, type, type}); } template - void flat() { + void flat(const TypePtr& type = CppToType::create()) { vector_size_t size = 20; // {0, null, null, 300, null, null, 600, null, null, ...}. auto first = makeFlatVector( size, [](vector_size_t row) { return row * 100; }, - [](vector_size_t row) { return row % 3 != 0; }); + [](vector_size_t row) { return row % 3 != 0; }, + type); // {0, 10, null, 30, 40, null, 60, 70, null, ...}. auto second = makeFlatVector( - size, [](vector_size_t row) { return row * 10; }, nullEvery(3, 2)); + size, + [](vector_size_t row) { return row * 10; }, + nullEvery(3, 2), + type); // {0, 1, 2, 3, 4, 5, 6, 7, 8, ...}. - auto third = makeFlatVector(size, [](vector_size_t row) { return row; }); + auto third = makeFlatVector( + size, [](vector_size_t row) { return row; }, nullptr, type); auto data = makeRowVector({first, second, third}); @@ -223,17 +250,18 @@ class GreatestTest : public SparkFunctionBaseTest { } template - void constant() { + void constant(const TypePtr& type = CppToType::create()) { vector_size_t size = 20; // {0, null, null, 3, null, null, 6, null, null, ...}. auto first = makeFlatVector( size, [](vector_size_t row) { return row; }, - [](vector_size_t row) { return row % 3 != 0; }); + [](vector_size_t row) { return row % 3 != 0; }, + type); // {9, 9, 9, ...}. - auto second = makeConstant(9, size); + auto second = makeConstant(9, size, type); auto data = makeRowVector({first, second}); @@ -259,6 +287,9 @@ TEST_F(GreatestTest, integral) { flat(); constant(); + + flat(DATE()); + constant(DATE()); } TEST_F(GreatestTest, floating) { @@ -300,7 +331,8 @@ TEST_F(GreatestTest, timestamp) { } TEST_F(GreatestTest, date) { - EXPECT_EQ(greatest(Date(100), Date(1000), Date(10000)), Date(10000)); + EXPECT_EQ(greatest(100, 1000, 10000, DATE()), 10000); } + } // namespace } // namespace facebook::velox::functions::sparksql::test diff --git a/velox/functions/sparksql/tests/SortArrayTest.cpp b/velox/functions/sparksql/tests/SortArrayTest.cpp index cea47a69958..f5313b32b86 100644 --- a/velox/functions/sparksql/tests/SortArrayTest.cpp +++ b/velox/functions/sparksql/tests/SortArrayTest.cpp @@ -127,12 +127,12 @@ TEST_F(SortArrayTest, timestamp) { } TEST_F(SortArrayTest, date) { - auto input = makeNullableArrayVector(dateInput()); + auto input = makeNullableArrayVector(dateInput(), ARRAY(DATE())); auto expected = dateAscNullSmallest(); testSortArray( input, - makeNullableArrayVector(expected), - makeNullableArrayVector(reverseNested(expected))); + makeNullableArrayVector(expected, ARRAY(DATE())), + makeNullableArrayVector(reverseNested(expected), ARRAY(DATE()))); } TEST_F(SortArrayTest, bool) { diff --git a/velox/row/UnsafeRowFast.cpp b/velox/row/UnsafeRowFast.cpp index 4d275523858..c8b0bbca542 100644 --- a/velox/row/UnsafeRowFast.cpp +++ b/velox/row/UnsafeRowFast.cpp @@ -97,8 +97,6 @@ void UnsafeRowFast::initialize(const TypePtr& type) { case TypeKind::REAL: FOLLY_FALLTHROUGH; case TypeKind::DOUBLE: - FOLLY_FALLTHROUGH; - case TypeKind::DATE: valueBytes_ = type->cppSizeInBytes(); fixedWidthTypeKind_ = true; supportsBulkCopy_ = decoded_.isIdentityMapping(); diff --git a/velox/serializers/PrestoSerializer.cpp b/velox/serializers/PrestoSerializer.cpp index 969921a994e..cbf4f0e44be 100644 --- a/velox/serializers/PrestoSerializer.cpp +++ b/velox/serializers/PrestoSerializer.cpp @@ -17,7 +17,6 @@ #include "velox/common/base/Crc.h" #include "velox/common/memory/ByteStream.h" #include "velox/functions/prestosql/types/TimestampWithTimeZoneType.h" -#include "velox/type/Date.h" #include "velox/vector/BiasVector.h" #include "velox/vector/ComplexVector.h" #include "velox/vector/FlatVector.h" @@ -87,6 +86,10 @@ bool isChecksumBitSet(int8_t codec) { } std::string typeToEncodingName(const TypePtr& type) { + if (type->isDate()) { + return "INT_ARRAY"; + } + switch (type->kind()) { case TypeKind::BOOLEAN: return "BYTE_ARRAY"; @@ -110,8 +113,6 @@ std::string typeToEncodingName(const TypePtr& type) { return "VARIABLE_WIDTH"; case TypeKind::TIMESTAMP: return "LONG_ARRAY"; - case TypeKind::DATE: - return "INT_ARRAY"; case TypeKind::ARRAY: return "ARRAY"; case TypeKind::MAP: @@ -232,36 +233,6 @@ void readLosslessTimestampValues( } } -Date readDate(ByteStream* source) { - int32_t days = source->read(); - return Date(days); -} - -template <> -void readValues( - ByteStream* source, - vector_size_t size, - BufferPtr nulls, - vector_size_t nullCount, - BufferPtr values) { - auto rawValues = values->asMutable(); - if (nullCount) { - int32_t toClear = 0; - bits::forEachSetBit(nulls->as(), 0, size, [&](int32_t row) { - // Set the values between the last non-null and this to type default. - for (; toClear < row; ++toClear) { - rawValues[toClear] = Date(); - } - rawValues[row] = readDate(source); - toClear = row + 1; - }); - } else { - for (int32_t row = 0; row < size; ++row) { - rawValues[row] = readDate(source); - } - } -} - int128_t readJavaDecimal(ByteStream* source) { constexpr int64_t kInt64DeserializeMask = ~(static_cast(1) << 63); // ByteStream does not support reading int128_t values. @@ -692,7 +663,6 @@ void readColumns( {TypeKind::REAL, &read}, {TypeKind::DOUBLE, &read}, {TypeKind::TIMESTAMP, &read}, - {TypeKind::DATE, &read}, {TypeKind::VARCHAR, &read}, {TypeKind::VARBINARY, &read}, {TypeKind::ARRAY, &readArrayVector}, @@ -947,13 +917,6 @@ void VectorStream::append(folly::Range values) { } } -template <> -void VectorStream::append(folly::Range values) { - for (auto& value : values) { - appendOne(value.days()); - } -} - template <> void VectorStream::append(folly::Range values) { // A bool constant is serialized via this. Accessing consecutive diff --git a/velox/serializers/tests/UnsafeRowSerializerTest.cpp b/velox/serializers/tests/UnsafeRowSerializerTest.cpp index 3d31bc429a1..f2e18fe219e 100644 --- a/velox/serializers/tests/UnsafeRowSerializerTest.cpp +++ b/velox/serializers/tests/UnsafeRowSerializerTest.cpp @@ -212,8 +212,8 @@ TEST_F(UnsafeRowSerializerTest, types) { TEST_F(UnsafeRowSerializerTest, date) { auto rowVector = makeRowVector({ - makeFlatVector({Date(0), Date(1)}), + makeFlatVector({0, 1}, DATE()), }); testRoundTrip(rowVector); -} +} \ No newline at end of file diff --git a/velox/substrait/SubstraitToVeloxExpr.cpp b/velox/substrait/SubstraitToVeloxExpr.cpp index ace7dafefb2..e110e6deb5e 100644 --- a/velox/substrait/SubstraitToVeloxExpr.cpp +++ b/velox/substrait/SubstraitToVeloxExpr.cpp @@ -71,11 +71,6 @@ Timestamp getLiteralValue(const ::substrait::Expression::Literal& literal) { return Timestamp::fromMicros(literal.timestamp()); } -template <> -Date getLiteralValue(const ::substrait::Expression::Literal& literal) { - return Date(literal.date()); -} - ArrayVectorPtr makeArrayVector(const VectorPtr& elements) { BufferPtr offsets = allocateOffsets(1, elements->pool()); BufferPtr sizes = allocateOffsets(1, elements->pool()); @@ -113,6 +108,9 @@ void setLiteralValue( } else { VELOX_FAIL("Unexpected string literal"); } + } else if (vector->type()->isDate()) { + auto dateVector = vector->template asFlatVector(); + dateVector->set(index, int(literal.date())); } else { vector->set(index, getLiteralValue(literal)); } @@ -250,7 +248,7 @@ SubstraitVeloxExprConverter::toVeloxExpr( } case ::substrait::Expression_Literal::LiteralTypeCase::kDate: return std::make_shared( - DATE(), variant(Date(substraitLit.date()))); + DATE(), variant(int(substraitLit.date()))); default: VELOX_NYI( "Substrait conversion not supported for type case '{}'", typeCase); @@ -298,7 +296,7 @@ ArrayVectorPtr SubstraitVeloxExprConverter::literalsToArrayVector( constructFlatVector, kind, listLiteral, childSize, veloxType, pool_)); } case ::substrait::Expression_Literal::LiteralTypeCase::kDate: - return makeArrayVector(constructFlatVector( + return makeArrayVector(constructFlatVector( listLiteral, childSize, DATE(), pool_)); case ::substrait::Expression_Literal::LiteralTypeCase::kTimestamp: return makeArrayVector(constructFlatVector( diff --git a/velox/substrait/TypeUtils.cpp b/velox/substrait/TypeUtils.cpp index 77cbd17d6df..add6d769220 100644 --- a/velox/substrait/TypeUtils.cpp +++ b/velox/substrait/TypeUtils.cpp @@ -68,8 +68,11 @@ std::string_view getNameBeforeDelimiter( TypePtr toVeloxType(const std::string& typeName) { VELOX_CHECK(!typeName.empty(), "Cannot convert empty string to Velox type."); - auto type = getNameBeforeDelimiter(typeName, "<"); - auto typeKind = mapNameToTypeKind(std::string(type)); + auto type = std::string(getNameBeforeDelimiter(typeName, "<")); + if (DATE()->toString() == type) { + return DATE(); + } + auto typeKind = mapNameToTypeKind(type); switch (typeKind) { case TypeKind::BOOLEAN: return BOOLEAN(); @@ -117,9 +120,6 @@ TypePtr toVeloxType(const std::string& typeName) { } return ROW(std::move(names), std::move(types)); } - case TypeKind::DATE: { - return DATE(); - } case TypeKind::UNKNOWN: return UNKNOWN(); default: diff --git a/velox/substrait/VeloxSubstraitSignature.cpp b/velox/substrait/VeloxSubstraitSignature.cpp index 162b0201983..d33cddee76b 100644 --- a/velox/substrait/VeloxSubstraitSignature.cpp +++ b/velox/substrait/VeloxSubstraitSignature.cpp @@ -19,9 +19,12 @@ namespace facebook::velox::substrait { -std::string VeloxSubstraitSignature::toSubstraitSignature( - const TypeKind typeKind) { - switch (typeKind) { +std::string VeloxSubstraitSignature::toSubstraitSignature(const TypePtr& type) { + if (type->isDate()) { + return "date"; + } + + switch (type->kind()) { case TypeKind::BOOLEAN: return "bool"; case TypeKind::TINYINT: @@ -42,8 +45,6 @@ std::string VeloxSubstraitSignature::toSubstraitSignature( return "vbin"; case TypeKind::TIMESTAMP: return "ts"; - case TypeKind::DATE: - return "date"; case TypeKind::ARRAY: return "list"; case TypeKind::MAP: @@ -55,7 +56,7 @@ std::string VeloxSubstraitSignature::toSubstraitSignature( default: VELOX_UNSUPPORTED( "Substrait type signature conversion not supported for type {}.", - mapTypeKindToName(typeKind)); + mapTypeKindToName(type->kind())); } } @@ -68,7 +69,7 @@ std::string VeloxSubstraitSignature::toSubstraitSignature( std::vector substraitTypeSignatures; substraitTypeSignatures.reserve(arguments.size()); for (const auto& type : arguments) { - substraitTypeSignatures.emplace_back(toSubstraitSignature(type->kind())); + substraitTypeSignatures.emplace_back(toSubstraitSignature(type)); } return functionName + ":" + folly::join("_", substraitTypeSignatures); } diff --git a/velox/substrait/VeloxSubstraitSignature.h b/velox/substrait/VeloxSubstraitSignature.h index 6f24312ea34..aa8304a4db5 100644 --- a/velox/substrait/VeloxSubstraitSignature.h +++ b/velox/substrait/VeloxSubstraitSignature.h @@ -29,7 +29,7 @@ class VeloxSubstraitSignature { /// name>:__..._ for more /// detail information about the argument type please refer to link /// https://substrait.io/extensions/#function-signature-compound-names. - static std::string toSubstraitSignature(const TypeKind typeKind); + static std::string toSubstraitSignature(const TypePtr& type); /// Given a velox scalar function name and argument types, return the /// substrait function signature. diff --git a/velox/substrait/VeloxToSubstraitExpr.cpp b/velox/substrait/VeloxToSubstraitExpr.cpp index f4eeada00d6..506c4023c76 100644 --- a/velox/substrait/VeloxToSubstraitExpr.cpp +++ b/velox/substrait/VeloxToSubstraitExpr.cpp @@ -166,10 +166,6 @@ const ::substrait::Expression_Literal& toSubstraitNotNullLiteral( literalExpr->set_timestamp(micros); break; } - case velox::TypeKind::DATE: { - literalExpr->set_date(variantValue.value().days()); - break; - } case velox::TypeKind::VARCHAR: { auto vCharValue = variantValue.value(); ::substrait::Expression_Literal::VarChar* sVarChar = @@ -280,17 +276,6 @@ const ::substrait::Expression_Literal& toSubstraitNotNullLiteral< return *literalExpr; } -template <> -const ::substrait::Expression_Literal& toSubstraitNotNullLiteral< - TypeKind::DATE>(google::protobuf::Arena& arena, const Date& value) { - ::substrait::Expression_Literal* literalExpr = - google::protobuf::Arena::CreateMessage<::substrait::Expression_Literal>( - &arena); - literalExpr->set_date(value.days()); - literalExpr->set_nullable(false); - return *literalExpr; -} - template <> const ::substrait::Expression_Literal& toSubstraitNotNullLiteral( diff --git a/velox/substrait/VeloxToSubstraitType.cpp b/velox/substrait/VeloxToSubstraitType.cpp index 9f3fa3a13d2..22df20d583c 100644 --- a/velox/substrait/VeloxToSubstraitType.cpp +++ b/velox/substrait/VeloxToSubstraitType.cpp @@ -25,6 +25,16 @@ const ::substrait::Type& VeloxToSubstraitTypeConvertor::toSubstraitType( const velox::TypePtr& type) { ::substrait::Type* substraitType = google::protobuf::Arena::CreateMessage<::substrait::Type>(&arena); + + if (type->isDate()) { + auto substraitDate = + google::protobuf::Arena::CreateMessage<::substrait::Type_Date>(&arena); + substraitDate->set_nullability( + ::substrait::Type_Nullability_NULLABILITY_NULLABLE); + substraitType->set_allocated_date(substraitDate); + return *substraitType; + } + switch (type->kind()) { case velox::TypeKind::BOOLEAN: { auto substraitBool = @@ -168,15 +178,6 @@ const ::substrait::Type& VeloxToSubstraitTypeConvertor::toSubstraitType( substraitType->set_allocated_user_defined(substraitUserDefined); break; } - case velox::TypeKind::DATE: { - auto substraitDate = - google::protobuf::Arena::CreateMessage<::substrait::Type_Date>( - &arena); - substraitDate->set_nullability( - ::substrait::Type_Nullability_NULLABILITY_NULLABLE); - substraitType->set_allocated_date(substraitDate); - break; - } case velox::TypeKind::FUNCTION: case velox::TypeKind::OPAQUE: case velox::TypeKind::INVALID: diff --git a/velox/substrait/tests/FunctionTest.cpp b/velox/substrait/tests/FunctionTest.cpp index 6a7a6b8c43c..07683801b36 100644 --- a/velox/substrait/tests/FunctionTest.cpp +++ b/velox/substrait/tests/FunctionTest.cpp @@ -187,11 +187,13 @@ TEST_F(FunctionTest, setVectorFromVariants) { resultVec->asFlatVector()->valueAt(1).toString()); resultVec = setVectorFromVariants( - DATE(), {variant(Date(9020)), variant(Date(8875))}, pool_.get()); + DATE(), {variant(9020), variant(8875)}, pool_.get()); ASSERT_EQ( - "1994-09-12", resultVec->asFlatVector()->valueAt(0).toString()); + "1994-09-12", + DATE()->toString(resultVec->asFlatVector()->valueAt(0))); ASSERT_EQ( - "1994-04-20", resultVec->asFlatVector()->valueAt(1).toString()); + "1994-04-20", + DATE()->toString(resultVec->asFlatVector()->valueAt(1))); resultVec = setVectorFromVariants( INTERVAL_DAY_TIME(), {variant(9020LL), variant(8875LL)}, pool_.get()); diff --git a/velox/substrait/tests/VeloxSubstraitRoundTripTest.cpp b/velox/substrait/tests/VeloxSubstraitRoundTripTest.cpp index 149342bc065..ca2435ebce2 100644 --- a/velox/substrait/tests/VeloxSubstraitRoundTripTest.cpp +++ b/velox/substrait/tests/VeloxSubstraitRoundTripTest.cpp @@ -465,7 +465,7 @@ TEST_F(VeloxSubstraitRoundTripTest, arrayLiteral) { makeArrayVector({{StringView("6")}})), makeConstantExpr(makeArrayVector( {{Timestamp(123'456, 123'000)}})), - makeConstantExpr(makeArrayVector({{Date(8035)}})), + makeConstantExpr(makeArrayVector({{8035}}, DATE())), makeConstantExpr(makeArrayVector( {{54 * 1000}}, INTERVAL_DAY_TIME())), makeConstantExpr(makeArrayVector({{}})), @@ -495,7 +495,7 @@ TEST_F(VeloxSubstraitRoundTripTest, arrayLiteral) { TEST_F(VeloxSubstraitRoundTripTest, dateType) { auto a = makeFlatVector({0, 1}); auto b = makeFlatVector({0.3, 0.4}); - auto c = makeFlatVector({Date(8036), Date(8035)}); + auto c = makeFlatVector({8036, 8035}, DATE()); auto vectors = makeRowVector({"a", "b", "c"}, {a, b, c}); createDuckDbTable({vectors}); diff --git a/velox/substrait/tests/VeloxSubstraitSignatureTest.cpp b/velox/substrait/tests/VeloxSubstraitSignatureTest.cpp index 941d7812907..9bcd037ef75 100644 --- a/velox/substrait/tests/VeloxSubstraitSignatureTest.cpp +++ b/velox/substrait/tests/VeloxSubstraitSignatureTest.cpp @@ -31,7 +31,7 @@ class VeloxSubstraitSignatureTest : public ::testing::Test { } static std::string toSubstraitSignature(const TypePtr& type) { - return VeloxSubstraitSignature::toSubstraitSignature(type->kind()); + return VeloxSubstraitSignature::toSubstraitSignature(type); } static std::string toSubstraitSignature( diff --git a/velox/tpch/gen/TpchGen.cpp b/velox/tpch/gen/TpchGen.cpp index 4ccd3ee2559..38fb06b9f65 100644 --- a/velox/tpch/gen/TpchGen.cpp +++ b/velox/tpch/gen/TpchGen.cpp @@ -82,11 +82,10 @@ double decimalToDouble(int64_t value) { return (double)value * 0.01; } -Date toDate(std::string_view stringDate) { - Date date; - parseTo(stringDate, date); - return date; +int32_t toDate(std::string_view stringDate) { + return DATE()->toDays(stringDate); } + } // namespace std::string_view toTableName(Table table) { @@ -371,7 +370,7 @@ RowVectorPtr genTpchOrders( auto custKeyVector = children[1]->asFlatVector(); auto orderStatusVector = children[2]->asFlatVector(); auto totalPriceVector = children[3]->asFlatVector(); - auto orderDateVector = children[4]->asFlatVector(); + auto orderDateVector = children[4]->asFlatVector(); auto orderPriorityVector = children[5]->asFlatVector(); auto clerkVector = children[6]->asFlatVector(); auto shipPriorityVector = children[7]->asFlatVector(); @@ -428,9 +427,9 @@ RowVectorPtr genTpchLineItem( auto returnFlagVector = children[8]->asFlatVector(); auto lineStatusVector = children[9]->asFlatVector(); - auto shipDateVector = children[10]->asFlatVector(); - auto commitDateVector = children[11]->asFlatVector(); - auto receiptDateVector = children[12]->asFlatVector(); + auto shipDateVector = children[10]->asFlatVector(); + auto commitDateVector = children[11]->asFlatVector(); + auto receiptDateVector = children[12]->asFlatVector(); auto shipInstructVector = children[13]->asFlatVector(); auto shipModeVector = children[14]->asFlatVector(); auto commentVector = children[15]->asFlatVector(); diff --git a/velox/tpch/gen/tests/TpchGenTest.cpp b/velox/tpch/gen/tests/TpchGenTest.cpp index f4b2f8b8e4d..39109f33123 100644 --- a/velox/tpch/gen/tests/TpchGenTest.cpp +++ b/velox/tpch/gen/tests/TpchGenTest.cpp @@ -210,16 +210,16 @@ TEST_F(TpchGenTestOrdersTest, batches) { auto orderKey = rowVector1->childAt(0)->asFlatVector(); auto orderTotalPrice = rowVector1->childAt(3)->asFlatVector(); - auto orderDate = rowVector1->childAt(4)->asFlatVector(); + auto orderDate = rowVector1->childAt(4)->asFlatVector(); EXPECT_EQ(1, orderKey->valueAt(0)); EXPECT_EQ(173665.47, orderTotalPrice->valueAt(0)); - EXPECT_EQ("1996-01-02", orderDate->valueAt(0).toString()); + EXPECT_EQ("1996-01-02", DATE()->toString(orderDate->valueAt(0))); LOG(INFO) << rowVector1->toString(0); EXPECT_EQ(40'000, orderKey->valueAt(9999)); EXPECT_EQ(87784.83, orderTotalPrice->valueAt(9999)); - EXPECT_EQ("1995-01-30", orderDate->valueAt(9999).toString()); + EXPECT_EQ("1995-01-30", DATE()->toString(orderDate->valueAt(9999))); LOG(INFO) << rowVector1->toString(9999); // Get second batch. @@ -230,16 +230,16 @@ TEST_F(TpchGenTestOrdersTest, batches) { orderKey = rowVector2->childAt(0)->asFlatVector(); orderTotalPrice = rowVector2->childAt(3)->asFlatVector(); - orderDate = rowVector2->childAt(4)->asFlatVector(); + orderDate = rowVector2->childAt(4)->asFlatVector(); EXPECT_EQ(40001, orderKey->valueAt(0)); EXPECT_EQ(100589.02, orderTotalPrice->valueAt(0)); - EXPECT_EQ("1995-02-25", orderDate->valueAt(0).toString()); + EXPECT_EQ("1995-02-25", DATE()->toString(orderDate->valueAt(0))); LOG(INFO) << rowVector2->toString(0); EXPECT_EQ(80000, orderKey->valueAt(9999)); EXPECT_EQ(142775.84, orderTotalPrice->valueAt(9999)); - EXPECT_EQ("1995-12-15", orderDate->valueAt(9999).toString()); + EXPECT_EQ("1995-12-15", DATE()->toString(orderDate->valueAt(9999))); LOG(INFO) << rowVector2->toString(9999); } @@ -320,15 +320,15 @@ TEST_F(TpchGenTestLineItemTest, batches) { EXPECT_LE(rowVector1->size(), ordersMaxSize * 7); auto orderKey = rowVector1->childAt(0)->asFlatVector(); - auto shipDate = rowVector1->childAt(10)->asFlatVector(); + auto shipDate = rowVector1->childAt(10)->asFlatVector(); EXPECT_EQ(1, orderKey->valueAt(0)); - EXPECT_EQ("1996-03-13", shipDate->valueAt(0).toString()); + EXPECT_EQ("1996-03-13", DATE()->toString(shipDate->valueAt(0))); LOG(INFO) << rowVector1->toString(0); vector_size_t lastRow = rowVector1->size() - 1; EXPECT_EQ(388, orderKey->valueAt(lastRow)); - EXPECT_EQ("1992-12-24", shipDate->valueAt(lastRow).toString()); + EXPECT_EQ("1992-12-24", DATE()->toString(shipDate->valueAt(lastRow))); LOG(INFO) << rowVector1->toString(lastRow); // Get next batch. @@ -339,15 +339,15 @@ TEST_F(TpchGenTestLineItemTest, batches) { EXPECT_LE(rowVector2->size(), ordersMaxSize * 7); orderKey = rowVector2->childAt(0)->asFlatVector(); - shipDate = rowVector2->childAt(10)->asFlatVector(); + shipDate = rowVector2->childAt(10)->asFlatVector(); EXPECT_EQ(389, orderKey->valueAt(0)); - EXPECT_EQ("1994-04-13", shipDate->valueAt(0).toString()); + EXPECT_EQ("1994-04-13", DATE()->toString(shipDate->valueAt(0))); LOG(INFO) << rowVector2->toString(0); lastRow = rowVector2->size() - 1; EXPECT_EQ(800, orderKey->valueAt(lastRow)); - EXPECT_EQ("1998-07-23", shipDate->valueAt(lastRow).toString()); + EXPECT_EQ("1998-07-23", DATE()->toString(shipDate->valueAt(lastRow))); LOG(INFO) << rowVector2->toString(lastRow); } diff --git a/velox/type/CMakeLists.txt b/velox/type/CMakeLists.txt index 95596f94c12..33aa8677fd4 100644 --- a/velox/type/CMakeLists.txt +++ b/velox/type/CMakeLists.txt @@ -18,7 +18,6 @@ add_subdirectory(tz) add_library( velox_type - Date.cpp DecimalUtil.cpp DoubleUtil.cpp Filter.cpp diff --git a/velox/type/Conversions.h b/velox/type/Conversions.h index 492ed3c730e..e86e303b1ee 100644 --- a/velox/type/Conversions.h +++ b/velox/type/Conversions.h @@ -56,10 +56,6 @@ struct Converter { return folly::to(v); } - static T cast(const Date&) { - VELOX_UNSUPPORTED("Conversion of Date to Boolean is not supported"); - } - static T cast(const Timestamp&) { VELOX_UNSUPPORTED("Conversion of Timestamp to Boolean is not supported"); } @@ -359,10 +355,6 @@ struct Converter< return static_cast(v); } - static T cast(const Date&) { - VELOX_UNSUPPORTED("Conversion of Date to Real or Double is not supported"); - } - static T cast(const Timestamp&) { VELOX_UNSUPPORTED( "Conversion of Timestamp to Real or Double is not supported"); @@ -426,45 +418,6 @@ struct Converter { return fromTimestampString(v.data(), v.size()); } - static T cast(const Date& d) { - static const int64_t kMillisPerDay{86'400'000}; - return Timestamp::fromMillis(d.days() * kMillisPerDay); - } -}; - -// Allow conversions from string to DATE type. -template -struct Converter { - using T = typename TypeTraits::NativeType; - template - static T cast(const From& /* v */) { - VELOX_UNSUPPORTED("Conversion to Date is not supported"); - return T(); - } - - static T cast(folly::StringPiece v) { - return fromDateString(v.data(), v.size()); - } - - static T cast(const StringView& v) { - return fromDateString(v.data(), v.size()); - } - - static T cast(const std::string& v) { - return fromDateString(v.data(), v.size()); - } - - static T cast(const Timestamp& t) { - static const int32_t kSecsPerDay{86'400}; - auto seconds = t.getSeconds(); - if (seconds >= 0 || seconds % kSecsPerDay == 0) { - return Date(seconds / kSecsPerDay); - } - // For division with negatives, minus 1 to compensate the discarded - // fractional part. e.g. -1/86'400 yields 0, yet it should be considered as - // -1 day. - return Date(seconds / kSecsPerDay - 1); - } }; } // namespace facebook::velox::util diff --git a/velox/type/Date.cpp b/velox/type/Date.cpp deleted file mode 100644 index 7f8f55e25c8..00000000000 --- a/velox/type/Date.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "velox/type/Date.h" -#include "velox/type/TimestampConversion.h" - -namespace facebook::velox { - -void parseTo(folly::StringPiece in, Date& out) { - auto daysSinceEpoch = util::fromDateString(in.data(), in.size()); - out = Date(daysSinceEpoch); -} - -std::string Date::toString() const { - // Find the number of seconds for the days_; - // Casting 86400 to int64 to handle overflows gracefully. - int64_t daySeconds = days_ * (int64_t)(86400); - auto tmValue = gmtime((const time_t*)&daySeconds); - if (!tmValue) { - VELOX_FAIL("Can't convert days to dates: {}", days_); - } - - // return ISO 8601 time format. - // %F - equivalent to "%Y-%m-%d" (the ISO 8601 date format) - std::ostringstream oss; - oss << std::put_time(tmValue, "%F"); - return oss.str(); -} - -} // namespace facebook::velox - -namespace std { -std::string to_string(const ::facebook::velox::Date& date) { - return date.toString(); -} - -} // namespace std diff --git a/velox/type/Date.h b/velox/type/Date.h deleted file mode 100644 index 9b9478f7751..00000000000 --- a/velox/type/Date.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) Facebook, Inc. and its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include "velox/common/base/Exceptions.h" -#include "velox/type/StringView.h" - -#include -#include -#include - -#include - -namespace facebook::velox { - -struct Date { - public: - constexpr Date() : days_(0) {} - constexpr Date(int32_t days) : days_(days) {} - - int32_t days() const { - return days_; - } - - void addDays(int32_t days) { - days_ += days; - } - - bool operator==(const Date& other) const { - return days_ == other.days_; - } - - bool operator!=(const Date& other) const { - return days_ != other.days_; - } - - bool operator<(const Date& other) const { - return days_ < other.days_; - } - - bool operator<=(const Date& other) const { - return days_ <= other.days_; - } - - bool operator>(const Date& other) const { - return days_ > other.days_; - } - - bool operator>=(const Date& other) const { - return days_ >= other.days_; - } - - // Needed for serialization of FlatVector - operator StringView() const {VELOX_NYI()} - - std::string toString() const; - - operator std::string() const { - return toString(); - } - - operator folly::dynamic() const { - return folly::dynamic(days_); - } - - private: - // Number of days since the epoch ( 1970-01-01). - int32_t days_; -}; - -void parseTo(folly::StringPiece in, ::facebook::velox::Date& out); - -template -void toAppend(const ::facebook::velox::Date& value, T* result) { - result->append(value.toString()); -} - -} // namespace facebook::velox - -namespace std { -template <> -struct hash<::facebook::velox::Date> { - size_t operator()(const ::facebook::velox::Date& value) const { - return std::hash{}(value.days()); - } -}; - -std::string to_string(const ::facebook::velox::Date& ts); - -} // namespace std - -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext& ctx) const { - return ctx.begin(); - } - - template - auto format(const facebook::velox::Date& d, FormatContext& ctx) const { - return fmt::format_to(ctx.out(), "{}", std::to_string(d)); - } -}; - -namespace folly { -template <> -struct hasher<::facebook::velox::Date> { - size_t operator()(const ::facebook::velox::Date& value) const { - return std::hash{}(value.days()); - } -}; - -} // namespace folly diff --git a/velox/type/Filter.h b/velox/type/Filter.h index f9e8a8733ae..f918ea61be6 100644 --- a/velox/type/Filter.h +++ b/velox/type/Filter.h @@ -28,7 +28,6 @@ #include "velox/common/base/Exceptions.h" #include "velox/common/base/SimdUtil.h" #include "velox/common/serialization/Serializable.h" -#include "velox/type/Date.h" #include "velox/type/StringView.h" #include "velox/type/Type.h" @@ -1903,8 +1902,6 @@ template static inline bool applyFilter(TFilter& filter, T value) { if constexpr (std::is_same_v) { return filter.testInt128(value); - } else if constexpr (std::is_same_v) { - return filter.testInt64(value.days()); } else if constexpr ( std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { diff --git a/velox/type/Type.cpp b/velox/type/Type.cpp index 257584c20cc..9e93d2d339e 100644 --- a/velox/type/Type.cpp +++ b/velox/type/Type.cpp @@ -21,6 +21,7 @@ #include #include #include "velox/common/base/Exceptions.h" +#include "velox/type/TimestampConversion.h" namespace std { template <> @@ -55,7 +56,6 @@ const std::unordered_map& getTypeStringMap() { {"VARCHAR", TypeKind::VARCHAR}, {"VARBINARY", TypeKind::VARBINARY}, {"TIMESTAMP", TypeKind::TIMESTAMP}, - {"DATE", TypeKind::DATE}, {"ARRAY", TypeKind::ARRAY}, {"MAP", TypeKind::MAP}, {"ROW", TypeKind::ROW}, @@ -99,7 +99,6 @@ std::string mapTypeKindToName(const TypeKind& typeKind) { {TypeKind::VARCHAR, "VARCHAR"}, {TypeKind::VARBINARY, "VARBINARY"}, {TypeKind::TIMESTAMP, "TIMESTAMP"}, - {TypeKind::DATE, "DATE"}, {TypeKind::ARRAY, "ARRAY"}, {TypeKind::MAP, "MAP"}, {TypeKind::ROW, "ROW"}, @@ -216,6 +215,7 @@ void Type::registerSerDe() { registry.Register("IntervalDayTimeType", IntervalDayTimeType::deserialize); registry.Register( "IntervalYearMonthType", IntervalYearMonthType::deserialize); + registry.Register("DateType", DateType::deserialize); } std::string ArrayType::toString() const { @@ -663,7 +663,6 @@ KOSKI_DEFINE_SCALAR_ACCESSOR(DOUBLE); KOSKI_DEFINE_SCALAR_ACCESSOR(TIMESTAMP); KOSKI_DEFINE_SCALAR_ACCESSOR(VARCHAR); KOSKI_DEFINE_SCALAR_ACCESSOR(VARBINARY); -KOSKI_DEFINE_SCALAR_ACCESSOR(DATE); KOSKI_DEFINE_SCALAR_ACCESSOR(UNKNOWN); #undef KOSKI_DEFINE_SCALAR_ACCESSOR @@ -823,8 +822,6 @@ TypePtr fromKindToScalerType(TypeKind kind) { return TIMESTAMP(); case TypeKind::DOUBLE: return DOUBLE(); - case TypeKind::DATE: - return DATE(); case TypeKind::UNKNOWN: return UNKNOWN(); default: @@ -910,6 +907,30 @@ std::string IntervalYearMonthType::valueToString(int32_t value) const { return oss.str(); } +std::string DateType::toString(int32_t days) const { + // Find the number of seconds for the days_; + // Casting 86400 to int64 to handle overflows gracefully. + int64_t daySeconds = days * (int64_t)(86400); + auto tmValue = gmtime((const time_t*)&daySeconds); + if (!tmValue) { + VELOX_FAIL("Can't convert days to dates: {}", days); + } + + // return ISO 8601 time format. + // %F - equivalent to "%Y-%m-%d" (the ISO 8601 date format) + std::ostringstream oss; + oss << std::put_time(tmValue, "%F"); + return oss.str(); +} + +int32_t DateType::toDays(folly::StringPiece in) const { + return util::fromDateString(in.data(), in.size()); +} + +int32_t DateType::toDays(const char* in, size_t len) const { + return util::fromDateString(in, len); +} + namespace { using SingletonTypeMap = std::unordered_map; @@ -925,9 +946,9 @@ const SingletonTypeMap& singletonBuiltInTypes() { {"VARCHAR", VARCHAR()}, {"VARBINARY", VARBINARY()}, {"TIMESTAMP", TIMESTAMP()}, - {"DATE", DATE()}, {"INTERVAL DAY TO SECOND", INTERVAL_DAY_TIME()}, {"INTERVAL YEAR TO MONTH", INTERVAL_YEAR_MONTH()}, + {"DATE", DATE()}, {"UNKNOWN", UNKNOWN()}, }; return kTypes; diff --git a/velox/type/Type.h b/velox/type/Type.h index 8d838b60fc2..606c254fa8c 100644 --- a/velox/type/Type.h +++ b/velox/type/Type.h @@ -36,7 +36,6 @@ #include "folly/CPortability.h" #include "velox/common/base/ClassName.h" #include "velox/common/serialization/Serializable.h" -#include "velox/type/Date.h" #include "velox/type/HugeInt.h" #include "velox/type/StringView.h" #include "velox/type/Timestamp.h" @@ -48,7 +47,7 @@ using int128_t = __int128_t; /// Velox type system supports a small set of SQL-compatible composeable types: /// BOOLEAN, TINYINT, SMALLINT, INTEGER, BIGINT, HUGEINT, REAL, DOUBLE, VARCHAR, -/// VARBINARY, TIMESTAMP, DATE, ARRAY, MAP, ROW +/// VARBINARY, TIMESTAMP, ARRAY, MAP, ROW /// /// This file has multiple C++ type definitions for each of these logical types. /// These logical definitions each serve slightly different purposes. @@ -71,8 +70,7 @@ enum class TypeKind : int8_t { VARCHAR = 7, VARBINARY = 8, TIMESTAMP = 9, - DATE = 10, - HUGEINT = 11, + HUGEINT = 10, // Enum values for ComplexTypes start after 30 to leave // some values space to accommodate adding new scalar/native // types above. @@ -263,20 +261,6 @@ struct TypeTraits { static constexpr const char* name = "TIMESTAMP"; }; -// Date is internally an int32_t that represents the days since the epoch -template <> -struct TypeTraits { - using ImplType = ScalarType; - using NativeType = Date; - using DeepCopiedType = Date; - static constexpr uint32_t minSubTypes = 0; - static constexpr uint32_t maxSubTypes = 0; - static constexpr TypeKind typeKind = TypeKind::DATE; - static constexpr bool isPrimitiveType = true; - static constexpr bool isFixedWidth = true; - static constexpr const char* name = "DATE"; -}; - template <> struct TypeTraits { using ImplType = ScalarType; @@ -539,7 +523,6 @@ class Type : public Tree>, VELOX_FLUENT_CAST(Varchar, VARCHAR) VELOX_FLUENT_CAST(Varbinary, VARBINARY) VELOX_FLUENT_CAST(Timestamp, TIMESTAMP) - VELOX_FLUENT_CAST(Date, DATE) VELOX_FLUENT_CAST(Array, ARRAY) VELOX_FLUENT_CAST(Map, MAP) VELOX_FLUENT_CAST(Row, ROW) @@ -555,6 +538,8 @@ class Type : public Tree>, bool isIntervalDayTime() const; + bool isDate() const; + bool containsUnknown() const; protected: @@ -1062,7 +1047,6 @@ using DoubleType = ScalarType; using TimestampType = ScalarType; using VarcharType = ScalarType; using VarbinaryType = ScalarType; -using DateType = ScalarType; constexpr long kMillisInSecond = 1000; constexpr long kMillisInMinute = 60 * kMillisInSecond; @@ -1175,6 +1159,60 @@ FOLLY_ALWAYS_INLINE bool Type::isIntervalYearMonth() const { return (this == INTERVAL_YEAR_MONTH().get()); } +/// Date is represented as the number of days since epoch start using int32_t. +class DateType : public IntegerType { + private: + DateType() = default; + + public: + static const std::shared_ptr& get() { + static const std::shared_ptr kType{new DateType()}; + return kType; + } + + const char* name() const override { + return "DATE"; + } + + bool equivalent(const Type& other) const override { + return this == &other; + } + + std::string toString() const override { + return name(); + } + + std::string toString(int32_t days) const; + + int32_t toDays(folly::StringPiece in) const; + + int32_t toDays(const char* in, size_t len) const; + + folly::dynamic serialize() const override { + folly::dynamic obj = folly::dynamic::object; + obj["name"] = "DateType"; + obj["type"] = name(); + return obj; + } + + static TypePtr deserialize(const folly::dynamic& /*obj*/) { + return DateType::get(); + } +}; + +FOLLY_ALWAYS_INLINE std::shared_ptr DATE() { + return DateType::get(); +} + +FOLLY_ALWAYS_INLINE bool isDateName(const std::string& name) { + return (name == DateType::get()->name()); +} + +FOLLY_ALWAYS_INLINE bool Type::isDate() const { + // The pointers can be compared since DATE is a singleton. + return this == DATE().get(); +} + /// Used as T for SimpleVector subclasses that wrap another vector when /// the wrapped vector is of a complex type. Applies to /// DictionaryVector, SequenceVector and ConstantVector. This must have @@ -1302,9 +1340,6 @@ std::shared_ptr OPAQUE() { return TEMPLATE_FUNC<::facebook::velox::TypeKind::TIMESTAMP>( \ __VA_ARGS__); \ } \ - case ::facebook::velox::TypeKind::DATE: { \ - return TEMPLATE_FUNC<::facebook::velox::TypeKind::DATE>(__VA_ARGS__); \ - } \ default: \ VELOX_FAIL( \ "not a scalar type! kind: {}", mapTypeKindToName(typeKind)); \ @@ -1359,10 +1394,6 @@ std::shared_ptr OPAQUE() { return TEMPLATE_FUNC( \ __VA_ARGS__); \ } \ - case ::facebook::velox::TypeKind::DATE: { \ - return TEMPLATE_FUNC( \ - __VA_ARGS__); \ - } \ default: \ VELOX_FAIL( \ "not a scalar type! kind: {}", mapTypeKindToName(typeKind)); \ @@ -1427,9 +1458,6 @@ std::shared_ptr OPAQUE() { return PREFIX<::facebook::velox::TypeKind::TIMESTAMP> SUFFIX( \ __VA_ARGS__); \ } \ - case ::facebook::velox::TypeKind::DATE: { \ - return PREFIX<::facebook::velox::TypeKind::DATE> SUFFIX(__VA_ARGS__); \ - } \ case ::facebook::velox::TypeKind::ARRAY: { \ return PREFIX<::facebook::velox::TypeKind::ARRAY> SUFFIX(__VA_ARGS__); \ } \ @@ -1515,9 +1543,6 @@ std::shared_ptr OPAQUE() { case ::facebook::velox::TypeKind::TIMESTAMP: { \ return CLASS<::facebook::velox::TypeKind::TIMESTAMP>::FIELD; \ } \ - case ::facebook::velox::TypeKind::DATE: { \ - return CLASS<::facebook::velox::TypeKind::DATE>::FIELD; \ - } \ case ::facebook::velox::TypeKind::ARRAY: { \ return CLASS<::facebook::velox::TypeKind::ARRAY>::FIELD; \ } \ @@ -1558,7 +1583,6 @@ VELOX_SCALAR_ACCESSOR(DOUBLE); VELOX_SCALAR_ACCESSOR(TIMESTAMP); VELOX_SCALAR_ACCESSOR(VARCHAR); VELOX_SCALAR_ACCESSOR(VARBINARY); -VELOX_SCALAR_ACCESSOR(DATE); VELOX_SCALAR_ACCESSOR(UNKNOWN); template @@ -1740,6 +1764,11 @@ struct IntervalYearMonth { IntervalYearMonth() {} }; +struct Date { + private: + Date() {} +}; + struct Varbinary { private: Varbinary() {} @@ -1787,7 +1816,9 @@ template <> struct SimpleTypeTrait : public TypeTraits {}; template <> -struct SimpleTypeTrait : public TypeTraits {}; +struct SimpleTypeTrait : public SimpleTypeTrait { + static constexpr const char* name = "DATE"; +}; template <> struct SimpleTypeTrait : public SimpleTypeTrait { @@ -1919,7 +1950,7 @@ template <> struct CppToType : public CppToTypeBase {}; template <> -struct CppToType : public CppToTypeBase {}; +struct CppToType : public CppToTypeBase {}; template struct CppToType> : public CppToTypeBase {}; diff --git a/velox/type/Variant.cpp b/velox/type/Variant.cpp index 3909d679d6d..4c6879612c7 100644 --- a/velox/type/Variant.cpp +++ b/velox/type/Variant.cpp @@ -69,19 +69,6 @@ struct VariantEquality { } }; -// date -template <> -struct VariantEquality { - template - static bool equals(const variant& a, const variant& b) { - if (a.isNull() || b.isNull()) { - return evaluateNullEquality(a, b); - } else { - return a.value() == b.value(); - } - } -}; - // array template <> struct VariantEquality { @@ -196,6 +183,10 @@ std::string variant::toJson(const TypePtr& type) const { return "null"; } + if (type->isDate()) { + return '"' + DATE()->toString(value()) + '"'; + } + switch (kind_) { case TypeKind::MAP: { auto& map = value(); @@ -299,10 +290,6 @@ std::string variant::toJson(const TypePtr& type) const { auto& timestamp = value(); return '"' + timestamp.toString() + '"'; } - case TypeKind::DATE: { - auto& date = value(); - return '"' + date.toString() + '"'; - } case TypeKind::OPAQUE: { // Return expression that we can't parse back - we use toJson for // debugging only. Variant::serialize should actually serialize the data. @@ -409,10 +396,6 @@ folly::dynamic variant::serialize() const { objValue = value(); break; } - case TypeKind::DATE: { - objValue = value(); - break; - } case TypeKind::OPAQUE: { serializeOpaque(variantObj, value()); break; @@ -521,9 +504,6 @@ variant variant::create(const folly::dynamic& variantobj) { case TypeKind::OPAQUE: { return deserializeOpaque(variantobj); } - case TypeKind::DATE: { - return variant::create(obj.asInt()); - } case TypeKind::TIMESTAMP: { return variant::create(Timestamp( variantobj["seconds"].asInt(), variantobj["nanos"].asInt())); @@ -583,10 +563,6 @@ uint64_t variant::hash() const { } return hash; } - case TypeKind::DATE: { - auto dateValue = value(); - return folly::Hash{}(dateValue.days()); - } case TypeKind::TIMESTAMP: { auto timestampValue = value(); return folly::Hash{}( diff --git a/velox/type/Variant.h b/velox/type/Variant.h index 6bd2441cdf8..3780159e35e 100644 --- a/velox/type/Variant.h +++ b/velox/type/Variant.h @@ -45,9 +45,6 @@ struct VariantEquality; template <> struct VariantEquality; -template <> -struct VariantEquality; - template <> struct VariantEquality; @@ -211,7 +208,6 @@ class variant { VELOX_VARIANT_SCALAR_MEMBERS(TypeKind::VARCHAR); // VARBINARY conflicts with VARCHAR, so we don't gen these methods // VELOX_VARIANT_SCALAR_MEMBERS(TypeKind::VARBINARY); - VELOX_VARIANT_SCALAR_MEMBERS(TypeKind::DATE) VELOX_VARIANT_SCALAR_MEMBERS(TypeKind::TIMESTAMP) VELOX_VARIANT_SCALAR_MEMBERS(TypeKind::UNKNOWN) #undef VELOX_VARIANT_SCALAR_MEMBERS @@ -263,13 +259,6 @@ class variant { input}}; } - static variant date(const Date& input) { - return { - TypeKind::DATE, - new - typename detail::VariantTypeTraits::stored_type{input}}; - } - template static variant opaque(const std::shared_ptr& input) { VELOX_CHECK(input.get(), "Can't create a variant of nullptr opaque type"); @@ -607,7 +596,6 @@ struct VariantConverter { return convert(value); case TypeKind::VARBINARY: return convert(value); - case TypeKind::DATE: case TypeKind::TIMESTAMP: case TypeKind::HUGEINT: // Default date/timestamp conversion is prone to errors and implicit diff --git a/velox/type/tests/FilterTest.cpp b/velox/type/tests/FilterTest.cpp index 1dc19986d36..d3dff281fbd 100644 --- a/velox/type/tests/FilterTest.cpp +++ b/velox/type/tests/FilterTest.cpp @@ -1814,9 +1814,8 @@ TEST(FilterTest, hugeIntRange) { } TEST(FilterTest, dateRange) { - Date low = Date(1); - Date high = Date(100); - auto filter = between(low.days(), high.days()); - EXPECT_TRUE(applyFilter(*filter, Date(10))); - EXPECT_FALSE(applyFilter(*filter, Date(101))); -} + auto filter = + between(DATE()->toDays("1970-01-01"), DATE()->toDays("1980-01-01")); + EXPECT_TRUE(applyFilter(*filter, DATE()->toDays("1970-06-01"))); + EXPECT_FALSE(applyFilter(*filter, DATE()->toDays("1980-06-01"))); +} \ No newline at end of file diff --git a/velox/type/tests/TypeTest.cpp b/velox/type/tests/TypeTest.cpp index 049147778e1..4f5fca1d221 100644 --- a/velox/type/tests/TypeTest.cpp +++ b/velox/type/tests/TypeTest.cpp @@ -127,10 +127,15 @@ TEST(TypeTest, date) { EXPECT_EQ(date->toString(), "DATE"); EXPECT_EQ(date->size(), 0); EXPECT_THROW(date->childAt(0), std::invalid_argument); - EXPECT_EQ(date->kind(), TypeKind::DATE); - EXPECT_STREQ(date->kindName(), "DATE"); + EXPECT_EQ(date->kind(), TypeKind::INTEGER); + EXPECT_STREQ(date->kindName(), "INTEGER"); EXPECT_EQ(date->begin(), date->end()); + EXPECT_TRUE(date->kindEquals(INTEGER())); + EXPECT_NE(*date, *INTEGER()); + EXPECT_FALSE(date->equivalent(*INTEGER())); + EXPECT_FALSE(INTEGER()->equivalent(*date)); + testTypeSerde(date); } @@ -248,19 +253,15 @@ TEST(TypeTest, longDecimal) { } TEST(TypeTest, dateToString) { - Date epoch(0); - EXPECT_EQ(epoch.toString(), "1970-01-01"); + EXPECT_EQ(DATE()->toString(0), "1970-01-01"); // 50 years after epoch - Date jan2020(18262); - EXPECT_EQ(jan2020.toString(), "2020-01-01"); + EXPECT_EQ(DATE()->toString(18262), "2020-01-01"); - Date beforeEpoch(-5); - EXPECT_EQ(beforeEpoch.toString(), "1969-12-27"); + EXPECT_EQ(DATE()->toString(-5), "1969-12-27"); // 50 years before epoch - Date wayBeforeEpoch(-18262); - EXPECT_EQ(wayBeforeEpoch.toString(), "1920-01-02"); + EXPECT_EQ(DATE()->toString(-18262), "1920-01-02"); // Trying a very large -integer for boundary checks. Such values are tested in // ExpressionFuzzer. @@ -269,70 +270,36 @@ TEST(TypeTest, dateToString) { // for the number of years, so the eventual results might look like garbage. // However, they are consistent with presto java so keeping the same // implementation. - Date dateOverflow(-1855961014); - EXPECT_EQ(dateOverflow.toString(), "-5079479-05-03"); -} - -TEST(TypeTest, dateComparison) { - Date epoch(0); - Date beforeEpoch(-5); - Date jan2020(18262); - Date jan2020Copy(18262); - Date dec2019(18261); - - EXPECT_EQ(jan2020, jan2020Copy); - EXPECT_EQ(jan2020Copy, jan2020); - - EXPECT_NE(jan2020, dec2019); - EXPECT_NE(dec2019, jan2020); - EXPECT_NE(epoch, beforeEpoch); - - EXPECT_LT(dec2019, jan2020); - EXPECT_LT(beforeEpoch, epoch); - - EXPECT_LE(jan2020, jan2020Copy); - EXPECT_LE(dec2019, jan2020); - EXPECT_LE(beforeEpoch, epoch); - - EXPECT_GT(jan2020, dec2019); - EXPECT_GT(epoch, beforeEpoch); - - EXPECT_GE(jan2020, jan2020Copy); - EXPECT_GE(jan2020, dec2019); - EXPECT_GE(epoch, beforeEpoch); + EXPECT_EQ(DATE()->toString(-1855961014), "-5079479-05-03"); } TEST(TypeTest, parseStringToDate) { auto parseDate = [](const std::string& dateStr) { - Date returnDate; - parseTo(dateStr, returnDate); - return returnDate; + return DATE()->toDays(dateStr); }; // Epoch. - EXPECT_EQ(parseDate("1970-01-01").days(), 0); + EXPECT_EQ(parseDate("1970-01-01"), 0); // 50 years after epoch. - EXPECT_EQ(parseDate("2020-01-01").days(), 18262); + EXPECT_EQ(parseDate("2020-01-01"), 18262); // Before epoch. - EXPECT_EQ(parseDate("1969-12-27").days(), -5); + EXPECT_EQ(parseDate("1969-12-27"), -5); // 50 years before epoch. - EXPECT_EQ(parseDate("1920-01-02").days(), -18262); + EXPECT_EQ(parseDate("1920-01-02"), -18262); // Century before epoch. - EXPECT_EQ(parseDate("1812-04-15").days(), -57604); + EXPECT_EQ(parseDate("1812-04-15"), -57604); // Century after epoch. - EXPECT_EQ(parseDate("2135-11-09").days(), 60577); + EXPECT_EQ(parseDate("2135-11-09"), 60577); } TEST(TypeTest, dateFormat) { auto parseDate = [](const std::string& dateStr) { - Date returnDate; - parseTo(dateStr, returnDate); - return returnDate; + return DATE()->toString(DATE()->toDays(dateStr)); }; EXPECT_EQ(fmt::format("{}", parseDate("2015-12-24")), "2015-12-24"); @@ -634,7 +601,7 @@ TEST(TypeTest, cpp2Type) { EXPECT_EQ(*CppToType::create(), *DOUBLE()); EXPECT_EQ(*CppToType::create(), *BOOLEAN()); EXPECT_EQ(*CppToType::create(), *TIMESTAMP()); - EXPECT_EQ(*CppToType::create(), *DATE()); + EXPECT_EQ(*CppToType::create(), *INTEGER()); EXPECT_EQ(*CppToType>::create(), *ARRAY(INTEGER())); auto type = CppToType>>::create(); EXPECT_EQ(*type, *MAP(INTEGER(), MAP(BIGINT(), REAL()))); @@ -810,7 +777,6 @@ TEST(TypeTest, fromKindToScalerType) { TypeKind::VARCHAR, TypeKind::VARBINARY, TypeKind::TIMESTAMP, - TypeKind::DATE, TypeKind::UNKNOWN}) { SCOPED_TRACE(mapTypeKindToName(kind)); auto type = fromKindToScalerType(kind); diff --git a/velox/type/tests/VariantTest.cpp b/velox/type/tests/VariantTest.cpp index 31b6e0cacee..1176eda48b0 100644 --- a/velox/type/tests/VariantTest.cpp +++ b/velox/type/tests/VariantTest.cpp @@ -205,7 +205,6 @@ TEST(VariantTest, serialize) { testSerDe(variant(TypeKind::VARCHAR)); testSerDe(variant(TypeKind::VARBINARY)); testSerDe(variant(TypeKind::TIMESTAMP)); - testSerDe(variant(TypeKind::DATE)); testSerDe(variant(TypeKind::ARRAY)); testSerDe(variant(TypeKind::MAP)); testSerDe(variant(TypeKind::ROW)); @@ -221,7 +220,6 @@ TEST(VariantTest, serialize) { testSerDe(variant((double)1.234)); testSerDe(variant("This is a test.")); testSerDe(variant::binary("This is a test.")); - testSerDe(variant(Date(123))); testSerDe(variant(Timestamp(1, 2))); } diff --git a/velox/vector/VectorPool.cpp b/velox/vector/VectorPool.cpp index 6f3f30c88cd..cc5fc67aa0d 100644 --- a/velox/vector/VectorPool.cpp +++ b/velox/vector/VectorPool.cpp @@ -24,7 +24,7 @@ namespace { /// type is not supported, i.e. not a built-in singleton type. FOLLY_ALWAYS_INLINE int32_t toCacheIndex(const TypePtr& type) { static constexpr int32_t kNumCachedVectorTypes = - static_cast(TypeKind::DATE) + 1; + static_cast(TypeKind::HUGEINT) + 1; static std::array kSupportedTypes = { BOOLEAN().get(), diff --git a/velox/vector/VectorPool.h b/velox/vector/VectorPool.h index aa6320de798..b953c7177b4 100644 --- a/velox/vector/VectorPool.h +++ b/velox/vector/VectorPool.h @@ -63,7 +63,7 @@ class VectorPool { memory::MemoryPool* const pool_; static constexpr int32_t kNumCachedVectorTypes = - static_cast(TypeKind::DATE) + 1; + static_cast(TypeKind::HUGEINT) + 1; /// Caches of pre-allocated vectors indexed by typeKind. std::array vectors_; diff --git a/velox/vector/VectorTypeUtils.h b/velox/vector/VectorTypeUtils.h index a5f00a34550..370ae9f9b2f 100644 --- a/velox/vector/VectorTypeUtils.h +++ b/velox/vector/VectorTypeUtils.h @@ -37,13 +37,6 @@ struct KindToFlatVector { using HashRowType = Timestamp; }; -template <> -struct KindToFlatVector { - using type = FlatVector; - using WrapperType = Date; - using HashRowType = Date; -}; - template <> struct KindToFlatVector { using type = MapVector; diff --git a/velox/vector/arrow/Bridge.cpp b/velox/vector/arrow/Bridge.cpp index f9cb1da3ca0..2d29834f751 100644 --- a/velox/vector/arrow/Bridge.cpp +++ b/velox/vector/arrow/Bridge.cpp @@ -216,6 +216,11 @@ const char* exportArrowFormatStr( formatBuffer = fmt::format("d:{},{}", precision, scale); return formatBuffer.c_str(); } + + if (type->isDate()) { + return "tdD"; + } + switch (type->kind()) { // Scalar types. case TypeKind::BOOLEAN: @@ -243,8 +248,6 @@ const char* exportArrowFormatStr( // TODO: need to figure out how we'll map this since in Velox we currently // store timestamps as two int64s (epoch in sec and nanos). return "ttn"; // time64 [nanoseconds] - case TypeKind::DATE: - return "tdD"; // date32[days] // Complex/nested types. case TypeKind::ARRAY: static_assert(sizeof(vector_size_t) == 4); @@ -425,7 +428,6 @@ void exportFlat( case TypeKind::INTEGER: case TypeKind::BIGINT: case TypeKind::HUGEINT: - case TypeKind::DATE: case TypeKind::REAL: case TypeKind::DOUBLE: exportValues(vec, rows, out, pool, holder); diff --git a/velox/vector/arrow/tests/ArrowBridgeArrayTest.cpp b/velox/vector/arrow/tests/ArrowBridgeArrayTest.cpp index da093027471..565401b7b1b 100644 --- a/velox/vector/arrow/tests/ArrowBridgeArrayTest.cpp +++ b/velox/vector/arrow/tests/ArrowBridgeArrayTest.cpp @@ -40,11 +40,13 @@ void exportToArrow(const TypePtr& type, ArrowSchema& out) { class ArrowBridgeArrayExportTest : public testing::Test { protected: template - void testFlatVector(const std::vector>& inputData) { + void testFlatVector( + const std::vector>& inputData, + const TypePtr& type = CppToType::create()) { const bool isString = std::is_same_v or std::is_same_v; - auto flatVector = vectorMaker_.flatVectorNullable(inputData); + auto flatVector = vectorMaker_.flatVectorNullable(inputData, type); ArrowArray arrowArray; exportToArrow(flatVector, arrowArray, pool_.get()); @@ -308,14 +310,16 @@ TEST_F(ArrowBridgeArrayExportTest, flatDouble) { } TEST_F(ArrowBridgeArrayExportTest, flatDate) { - testFlatVector({ - std::numeric_limits::min(), - std::nullopt, - std::numeric_limits::max(), - std::numeric_limits::max(), - std::nullopt, - std::nullopt, - }); + testFlatVector( + { + std::numeric_limits::min(), + std::nullopt, + std::numeric_limits::max(), + std::numeric_limits::max(), + std::nullopt, + std::nullopt, + }, + DATE()); } TEST_F(ArrowBridgeArrayExportTest, flatString) { @@ -822,7 +826,7 @@ class ArrowBridgeArrayImportTest : public ArrowBridgeArrayExportTest { testArrowImport("s", {5, 4, 3, 1, 2}); testArrowImport("i", {5, 4, 3, 1, 2}); - testArrowImport("tdD", {5, 4, 3, 1, 2}); + testArrowImport("tdD", {5, 4, 3, 1, 2}); testArrowImport("l", {}); testArrowImport("l", {std::nullopt}); diff --git a/velox/vector/fuzzer/VectorFuzzer.cpp b/velox/vector/fuzzer/VectorFuzzer.cpp index e42397113c1..dbdaf737fcf 100644 --- a/velox/vector/fuzzer/VectorFuzzer.cpp +++ b/velox/vector/fuzzer/VectorFuzzer.cpp @@ -22,7 +22,6 @@ #include #include "velox/common/base/Exceptions.h" -#include "velox/type/Date.h" #include "velox/type/Timestamp.h" #include "velox/vector/FlatVector.h" #include "velox/vector/NullsBuilder.h" @@ -132,10 +131,6 @@ Timestamp randTimestamp(FuzzerGenerator& rng, VectorFuzzer::Options opts) { return {}; // no-op. } -Date randDate(FuzzerGenerator& rng) { - return Date(rand(rng)); -} - size_t getElementsVectorLength( const VectorFuzzer::Options& opts, vector_size_t size) { @@ -254,9 +249,6 @@ VectorPtr fuzzConstantPrimitiveImpl( if constexpr (std::is_same_v) { return std::make_shared>( pool, size, false, type, randTimestamp(rng, opts)); - } else if constexpr (std::is_same_v) { - return std::make_shared>( - pool, size, false, type, randDate(rng)); } else if (type->isShortDecimal()) { return std::make_shared>( pool, size, false, type, randShortDecimal(type, rng)); @@ -286,8 +278,6 @@ void fuzzFlatPrimitiveImpl( flatVector->set(i, randString(rng, opts, strBuf, converter)); } else if constexpr (std::is_same_v) { flatVector->set(i, randTimestamp(rng, opts)); - } else if constexpr (std::is_same_v) { - flatVector->set(i, randDate(rng)); } else if constexpr (std::is_same_v) { if (vector->type()->isShortDecimal()) { flatVector->set(i, randShortDecimal(vector->type(), rng)); @@ -791,7 +781,6 @@ TypePtr VectorFuzzer::randScalarNonFloatingPointType() { VARCHAR(), VARBINARY(), TIMESTAMP(), - DATE(), }; static constexpr int kNumTypes = sizeof(kNonFloatingPointTypes) / sizeof(kNonFloatingPointTypes[0]); @@ -812,7 +801,6 @@ TypePtr VectorFuzzer::randType(int maxDepth) { VARCHAR(), VARBINARY(), TIMESTAMP(), - DATE(), }; static constexpr int kNumScalarTypes = sizeof(kScalarTypes) / sizeof(kScalarTypes[0]); diff --git a/velox/vector/tests/VectorTest.cpp b/velox/vector/tests/VectorTest.cpp index 69d1ec069cc..ec0fefae203 100644 --- a/velox/vector/tests/VectorTest.cpp +++ b/velox/vector/tests/VectorTest.cpp @@ -884,11 +884,6 @@ Timestamp VectorTest::testValue(int32_t i, BufferPtr& /*space*/) { return Timestamp(i * 1000, (i % 1000) * 1000000); } -template <> -Date VectorTest::testValue(int32_t i, BufferPtr& /*space*/) { - return Date(i); -} - template <> std::shared_ptr VectorTest::testValue(int32_t i, BufferPtr& /*space*/) { return std::make_shared(i); @@ -957,7 +952,7 @@ TEST_F(VectorTest, createStr) { TEST_F(VectorTest, createOther) { testFlat(BOOLEAN(), vectorSize_); testFlat(TIMESTAMP(), vectorSize_); - testFlat(DATE(), vectorSize_); + testFlat(DATE(), vectorSize_); testFlat(INTERVAL_DAY_TIME(), vectorSize_); } @@ -1630,7 +1625,7 @@ TEST_F(VectorCreateConstantTest, null) { testNullConstant(DECIMAL(20, 5)); testNullConstant(TIMESTAMP()); - testNullConstant(DATE()); + testNullConstant(DATE()); testNullConstant(INTERVAL_DAY_TIME()); testNullConstant(VARCHAR()); diff --git a/velox/vector/tests/utils/VectorMaker.h b/velox/vector/tests/utils/VectorMaker.h index 798df4c251a..b342ea43101 100644 --- a/velox/vector/tests/utils/VectorMaker.h +++ b/velox/vector/tests/utils/VectorMaker.h @@ -328,7 +328,8 @@ class VectorMaker { std::function sizeAt, std::function valueAt, std::function isNullAt = nullptr, - std::function valueIsNullAt = nullptr) { + std::function valueIsNullAt = nullptr, + const TypePtr& arrayType = ARRAY(CppToType::create())) { BufferPtr nulls; BufferPtr offsets; BufferPtr sizes; @@ -337,7 +338,7 @@ class VectorMaker { return std::make_shared( pool_, - ARRAY(CppToType::create()), + arrayType, nulls, size, offsets, @@ -393,9 +394,9 @@ class VectorMaker { std::function sizeAt, std::function valueAt, - std::function isNullAt = nullptr) { - return arrayVectorImpl( - ARRAY(CppToType::create()), size, sizeAt, valueAt, isNullAt); + std::function isNullAt = nullptr, + const TypePtr& arrayType = ARRAY(CppToType::create())) { + return arrayVectorImpl(arrayType, size, sizeAt, valueAt, isNullAt); } template diff --git a/velox/vector/tests/utils/VectorTestBase.h b/velox/vector/tests/utils/VectorTestBase.h index 4f2bc4de4a0..68be06c40cc 100644 --- a/velox/vector/tests/utils/VectorTestBase.h +++ b/velox/vector/tests/utils/VectorTestBase.h @@ -448,8 +448,9 @@ class VectorTestBase { // }); template ArrayVectorPtr makeNullableArrayVector( - const std::vector>>>& data) { - return vectorMaker_.arrayVectorNullable(data); + const std::vector>>>& data, + const TypePtr& arrayType = ARRAY(CppToType::create())) { + return vectorMaker_.arrayVectorNullable(data, arrayType); } template @@ -458,9 +459,10 @@ class VectorTestBase { std::function sizeAt, std::function valueAt, std::function isNullAt = nullptr, - std::function valueIsNullAt = nullptr) { + std::function valueIsNullAt = nullptr, + const TypePtr& arrayType = ARRAY(CppToType::create())) { return vectorMaker_.arrayVector( - size, sizeAt, valueAt, isNullAt, valueIsNullAt); + size, sizeAt, valueAt, isNullAt, valueIsNullAt, arrayType); } template @@ -469,8 +471,10 @@ class VectorTestBase { std::function sizeAt, std::function valueAt, - std::function isNullAt = nullptr) { - return vectorMaker_.arrayVector(size, sizeAt, valueAt, isNullAt); + std::function isNullAt = nullptr, + const TypePtr& arrayType = ARRAY(CppToType::create())) { + return vectorMaker_.arrayVector( + size, sizeAt, valueAt, isNullAt, arrayType); } // Convenience function to create vector from a base vector.