Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions python/cudf/cudf/core/column/datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
get_dtype_of_same_kind,
)
from cudf.utils.scalar import pa_scalar_to_plc_scalar
from cudf.utils.temporal import (
_unit_to_name,
unit_to_nanoseconds_conversion,
)
from cudf.utils.utils import _EQUALITY_OPS, is_na_like

if TYPE_CHECKING:
Expand Down Expand Up @@ -533,6 +537,37 @@ def as_datetime_column(self, dtype: np.dtype) -> DatetimeColumn:
"Cannot use .astype to convert from timezone-naive dtype to timezone-aware dtype. "
"Use tz_localize instead."
)
target_unit = (
np.datetime_data(dtype)[0]
if isinstance(dtype, np.dtype)
else dtype.pyarrow_dtype.unit
)
if (
len(self) != self.null_count
and unit_to_nanoseconds_conversion[target_unit]
< unit_to_nanoseconds_conversion[self.time_unit]
):
# Casting to a finer resolution multiplies the underlying
# int64 values and can silently wrap around. pandas
# bounds-checks every narrowing conversion
# (astype_overflowsafe) and raises instead. Compare on the
# integer view: a host round-trip through pd.Timestamp can
# itself overflow for extreme values.
lo, hi = self.astype(np.dtype(np.int64)).minmax()
bound = np.iinfo(np.int64).max // (
unit_to_nanoseconds_conversion[self.time_unit]
// unit_to_nanoseconds_conversion[target_unit]
)
offender = None
if hi > bound:
offender = np.datetime64(int(hi), self.time_unit) # type: ignore[call-overload]
elif lo < -bound:
offender = np.datetime64(int(lo), self.time_unit) # type: ignore[call-overload]
if offender is not None:
raise pd.errors.OutOfBoundsDatetime(
f"Out of bounds {_unit_to_name[target_unit]} "
f"timestamp: {str(offender).replace('T', ' ')}"
)
return self.cast(dtype=dtype) # type: ignore[return-value]

def as_timedelta_column(self, dtype: np.dtype) -> None: # type: ignore[override]
Expand Down Expand Up @@ -977,7 +1012,9 @@ def tz_localize(
)

transition_times, offsets = _get_tz_data(tzname)
transition_times_local = (transition_times + offsets).astype(
# Use the raw cast: transition tables contain sentinel entries
# beyond the finer units' bounds that intentionally wrap around.
transition_times_local = (transition_times + offsets).cast(
localized.dtype
)
indices = (
Expand Down Expand Up @@ -1061,7 +1098,10 @@ def _local_time(self) -> DatetimeColumn:
transition_times, offsets = _get_tz_data(str(self.tz))
base_dtype = _get_base_dtype(self.dtype)
indices = (
transition_times.astype(base_dtype).searchsorted(
# Use the raw cast: transition tables contain sentinel
# entries beyond the finer units' bounds that intentionally
# wrap around.
transition_times.cast(base_dtype).searchsorted(
self._utc_time.astype(base_dtype), side="right"
)
- 1
Expand Down
32 changes: 31 additions & 1 deletion python/cudf/cudf/core/column/string.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@
is_pandas_nullable_extension_dtype,
)
from cudf.utils.scalar import pa_scalar_to_plc_scalar
from cudf.utils.temporal import infer_format
from cudf.utils.temporal import (
infer_format,
raise_if_datetime_seconds_out_of_bounds,
)
from cudf.utils.utils import is_na_like

if TYPE_CHECKING:
Expand Down Expand Up @@ -330,6 +333,33 @@ def strptime(
if not valid.all():
raise ValueError(f"Column contains invalid data for {format=}")

if isinstance(dtype, np.dtype):
target_unit = np.datetime_data(dtype)[0]
elif isinstance(dtype, pd.DatetimeTZDtype):
target_unit = dtype.unit
else:
target_unit = dtype.pyarrow_dtype.unit
if target_unit != "s" and len(without_nat):
# libcudf parses directly into int64 values of the
# target unit and silently wraps on overflow
# (see https://github.com/rapidsai/cudf/issues/23247).
# Parse to seconds first (which cannot realistically
# overflow) and reject values whose whole-second part
# falls outside the target unit's range, like pandas
# does. This double parse can be removed once libcudf
# detects the overflow itself.
with without_nat.access(mode="read", scope="internal"):
seconds = ColumnBase.create(
plc.strings.convert.convert_datetime.to_timestamps(
without_nat.plc_column,
dtype_to_pylibcudf_type(np.dtype("datetime64[s]")),
format,
),
np.dtype("datetime64[s]"),
)
lo, hi = seconds.minmax()
raise_if_datetime_seconds_out_of_bounds(lo, hi, target_unit)

casting_func = plc.strings.convert.convert_datetime.to_timestamps
add_back_nat = is_nat.any()
elif dtype.kind == "m":
Expand Down
13 changes: 12 additions & 1 deletion python/cudf/cudf/core/column/temporal_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
dtype_to_pylibcudf_type,
find_common_type,
)
from cudf.utils.temporal import unit_to_nanoseconds_conversion
from cudf.utils.utils import is_na_like


Expand Down Expand Up @@ -379,6 +380,14 @@ def find_and_replace(
def can_cast_safely(self, to_dtype: DtypeObj) -> bool:
if to_dtype.kind == self.dtype.kind:
to_res, _ = np.datetime_data(to_dtype)
if (
unit_to_nanoseconds_conversion[to_res]
> unit_to_nanoseconds_conversion[self.time_unit]
):
# Casting to a coarser resolution truncates any
# sub-resolution components; callers (replace, join key
# matching) rely on "safely" meaning lossless.
return False
max_val = self.max()
if isinstance(max_val, (pd.Timedelta, pd.Timestamp)):
max_val = max_val.to_numpy().astype(self.dtype)
Expand All @@ -403,7 +412,9 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool:
np.iinfo(self._UNDERLYING_DTYPE).max,
to_res, # type: ignore[call-overload]
).astype(f"m8[{self.time_unit}]", copy=False)
return bool(max_dist <= max_to_res and min_dist <= max_to_res)
# The negative bound is symmetric: int64 min is the NaT
# sentinel, so the valid range is +/-(2**63 - 1).
return bool(max_dist <= max_to_res and min_dist >= -max_to_res)
elif to_dtype == self._UNDERLYING_DTYPE or is_dtype_obj_string(
to_dtype
):
Expand Down
11 changes: 10 additions & 1 deletion python/cudf/cudf/core/indexing_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast

import numpy as np
import pandas as pd

import pylibcudf as plc

Expand Down Expand Up @@ -721,7 +722,15 @@ def parse_single_row_loc_key(
return MaskIndexer(BooleanMask(key, n))
elif index.dtype.kind == "M":
# Try to turn strings into datetimes
key = as_column(key, dtype=index.dtype)
try:
key = as_column(key, dtype=index.dtype)
except pd.errors.OutOfBoundsDatetime:
if is_scalar:
# A label beyond the index unit's bounds cannot
# be present; pandas raises KeyError for scalar
# lookups but OutOfBoundsDatetime for list keys.
raise KeyError(key.element_indexing(0))
raise
haystack = index._column
gather_map = ordered_find(key, haystack)
if is_scalar and len(gather_map.column) == 1:
Expand Down
20 changes: 13 additions & 7 deletions python/cudf/cudf/core/tools/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,17 +477,23 @@ def _process_col(
)
else:
if format is not None and "f" in format and unit is None:
col_ns = col.strptime(
dtype=np.dtype(_unit_dtype_map["ns"]), format=format
)
col_us = col.strptime(
dtype=np.dtype(_unit_dtype_map["us"]), format=format
)
res = col_ns != col_us
if res.any():
col = col_ns
else:
try:
col_ns = col.strptime(
dtype=np.dtype(_unit_dtype_map["ns"]), format=format
)
except pd.errors.OutOfBoundsDatetime:
# Values beyond the nanosecond Timestamp range: keep
# microsecond precision, like pandas' unit inference.
col = col_us
else:
res = col_ns != col_us
if res.any():
col = col_ns
else:
col = col_us
else:
if format is None:
format = infer_format(
Expand Down
16 changes: 0 additions & 16 deletions python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,22 +284,6 @@ def pytest_unconfigure(config):
"tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[False-min]": "TODO: Add a reason for failure",
"tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[True-max]": "TODO: Add a reason for failure",
"tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[True-min]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-array-datetime64[D]]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-array-object-datetime.datetime]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-array-object-numpy-scalar]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-array-object-string]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-dict-datetime64[D]]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-dict-object-datetime.datetime]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-dict-object-numpy-scalar]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[DataFrame-dict-object-string]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Index-datetime64[D]]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Index-object-datetime.datetime]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Index-object-numpy-scalar]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Index-object-string]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Series-datetime64[D]]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Series-object-datetime.datetime]": "TODO: Add a reason for failure",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Series-object-numpy-scalar]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_constructors.py::TestConstruction::test_constructor_datetime_outofbound[Series-object-string]": "Failed: DID NOT RAISE <class 'pandas.errors.OutOfBoundsDatetime'>",
"tests/base/test_conversion.py::test_array[index-arr3-_left]": "TODO: Add a reason for failure",
"tests/base/test_conversion.py::test_array[index-arr4-_sparse_values]": "TODO: Add a reason for failure",
"tests/base/test_conversion.py::test_array[series-arr4-_sparse_values]": "TODO: Add a reason for failure",
Expand Down
22 changes: 22 additions & 0 deletions python/cudf/cudf/tests/general_functions/test_to_datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,3 +453,25 @@ def test_to_datetime_rejects_non_datetime_dotted_string():
ValueError, match="Unable to infer the timestamp format"
):
cudf.to_datetime(cudf.Series(["1.2"]))


def test_to_datetime_out_of_bounds_nanosecond_precision_string():
# More than 6 fractional-second digits imply nanosecond precision,
# and year 2263 exceeds the nanosecond Timestamp range.
data = np.array(["2263-01-01 00:00:00.123456789"], dtype=object)
assert_exceptions_equal(
lfunc=pd.to_datetime,
rfunc=cudf.to_datetime,
lfunc_args_and_kwargs=([data],),
rfunc_args_and_kwargs=([data],),
)


def test_to_datetime_format_f_out_of_ns_bounds_keeps_us():
# With an explicit %f format, values beyond the nanosecond range
# keep microsecond precision like pandas' unit inference.
data = ["2263-01-01 00:00:00.123456"]
fmt = "%Y-%m-%d %H:%M:%S.%f"
expected = pd.to_datetime(data, format=fmt)
result = cudf.to_datetime(data, format=fmt)
assert_eq(result, expected)
10 changes: 10 additions & 0 deletions python/cudf/cudf/tests/private_objects/test_column.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,16 @@ def test_datetime_can_cast_safely():

assert sr._column.can_cast_safely(np.dtype("datetime64[ns]")) is False

# Pre-epoch values below the target range are also unsafe.
sr = cudf.Series(["1600-01-01", "2000-01-31"], dtype="datetime64[ms]")
assert sr._column.can_cast_safely(np.dtype("datetime64[ns]")) is False

# Casting to a coarser resolution truncates sub-resolution
# components, so it is not considered safe; equal resolution is.
sr = cudf.Series(["2000-01-31"], dtype="datetime64[ns]")
assert sr._column.can_cast_safely(np.dtype("datetime64[s]")) is False
assert sr._column.can_cast_safely(np.dtype("datetime64[ns]"))


@pytest.mark.parametrize(
"data_",
Expand Down
22 changes: 21 additions & 1 deletion python/cudf/cudf/tests/series/indexing/test_loc.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import cupy as cp
Expand Down Expand Up @@ -438,3 +438,23 @@ def test_loc_wrong_type_slice_datetimeindex():
)
with pytest.raises(TypeError):
ser_pd.loc[2:]


def test_loc_datetime_key_out_of_bounds():
# A label beyond the index unit's range cannot be present: pandas
# raises KeyError for scalar lookups and OutOfBoundsDatetime for
# list keys.
index = np.array(["2000-01-01", "2000-01-02"], dtype="datetime64[ns]")
ser_pd = pd.Series([1, 2], index=pd.Index(index))
ser_cudf = cudf.Series([1, 2], index=cudf.Index(index))
key = np.datetime64("9999-01-01", "s")

with pytest.raises(KeyError):
ser_pd.loc[key]
with pytest.raises(KeyError):
ser_cudf.loc[key]

with pytest.raises(pd.errors.OutOfBoundsDatetime):
ser_pd.loc[[key]]
with pytest.raises(pd.errors.OutOfBoundsDatetime):
ser_cudf.loc[[key]]
66 changes: 66 additions & 0 deletions python/cudf/cudf/tests/series/methods/test_astype.py
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,72 @@ def test_typecast_to_different_datetime_resolutions(datetime_types_as_str):
np.testing.assert_equal(np_data, gdf_series.to_numpy())


@pytest.mark.parametrize(
"data",
[
np.array(["2263-01-01"], dtype="datetime64[s]"),
np.array(["1600-01-01"], dtype="datetime64[s]"),
np.array(["2263-01-01", "NaT"], dtype="datetime64[s]"),
],
ids=["above-range", "below-range", "with-null"],
)
def test_typecast_datetime_narrowing_out_of_bounds(data):
# Casting to a finer resolution whose int64 range cannot hold the
# values must raise like pandas instead of wrapping around.
psr = pd.Series(data)
gsr = cudf.Series(data)
assert_exceptions_equal(
lfunc=psr.astype,
rfunc=gsr.astype,
lfunc_args_and_kwargs=(["datetime64[ns]"],),
rfunc_args_and_kwargs=(["datetime64[ns]"],),
)


def test_typecast_datetime_narrowing_in_bounds():
# The bounds check is relative to the target unit: year 9999 fits in
# seconds and microseconds, just not nanoseconds.
data = [datetime.date(9999, 12, 31)]
psr = pd.Series(data, dtype="datetime64[s]")
gsr = cudf.Series(data, dtype="datetime64[s]")
assert_eq(psr.astype("datetime64[us]"), gsr.astype("datetime64[us]"))

# All-null columns can always be narrowed.
all_null = np.array(["NaT", "NaT"], dtype="datetime64[s]")
assert_eq(
pd.Series(all_null).astype("datetime64[ns]"),
cudf.Series(all_null).astype("datetime64[ns]"),
)


def test_typecast_datetime_tz_narrowing_out_of_bounds():
data = np.array(["2263-01-01"], dtype="datetime64[s]")
psr = pd.Series(data).dt.tz_localize("UTC")
gsr = cudf.Series(data).dt.tz_localize("UTC")
target = pd.DatetimeTZDtype("ns", "UTC")
assert_exceptions_equal(
lfunc=psr.astype,
rfunc=gsr.astype,
lfunc_args_and_kwargs=([target],),
rfunc_args_and_kwargs=([target],),
)


def test_string_astype_datetime_tz():
target = pd.DatetimeTZDtype("ns", "UTC")

data = ["2000-01-01"]
assert_eq(pd.Series(data).astype(target), cudf.Series(data).astype(target))

out_of_bounds = ["2263-01-01"]
assert_exceptions_equal(
lfunc=pd.Series(out_of_bounds).astype,
rfunc=cudf.Series(out_of_bounds).astype,
lfunc_args_and_kwargs=([target],),
rfunc_args_and_kwargs=([target],),
)


@pytest.mark.parametrize(
"data",
[
Expand Down
15 changes: 14 additions & 1 deletion python/cudf/cudf/tests/series/methods/test_replace.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import re
Expand Down Expand Up @@ -501,3 +501,16 @@ def test_replace_timedelta_series():
)

assert_eq(pd_result, cudf_result)


def test_replace_datetime_sub_resolution_value_is_noop():
# The to_replace value has sub-second precision that cannot exist in
# a seconds-resolution column; it must not be truncated into a match.
data = np.array(["2000-01-01"], dtype="datetime64[s]")
pd_result = pd.Series(data).replace(
pd.Timestamp("2000-01-01 00:00:00.5"), pd.Timestamp("1999-01-01")
)
cudf_result = cudf.Series(data).replace(
pd.Timestamp("2000-01-01 00:00:00.5"), pd.Timestamp("1999-01-01")
)
assert_eq(pd_result, cudf_result)
Loading
Loading