From 9243c9b266a38f14d2e40f03f19e81837c5a5db6 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Thu, 24 Sep 2026 11:02:43 -0400 Subject: [PATCH 1/4] test: a zone on offset-less CSV text is attached, not converted (#231) Seven failing tests in test_tallyman_read_csv_contract.py, each importing a one-column CSV under a zoned timestamp schema and reading the snapshot back with pyarrow: - 09:30 under America/New_York is stored as 09:30-05:00 in January and 09:30-04:00 in July (today 04:30-05:00 and 05:30-04:00). - the same with timestamp('America/New_York', 9) keeps every ns digit (today the digits survive but the instant is shifted by five hours). - 2024-03-10 02:30, which New York skips, raises a ValueError naming the column, the value and its row (today it is stored as 21:30 the day before). - 2024-11-03 01:30, which New York passes twice, raises the same way (today it is stored as 21:30 the day before). - a column mixing text with and without a UTC offset raises an error that says so and names a row of each kind, whether the offset comes second, first, or 50,000 rows in (today a generic "the schema does not parse" error that suggests reading the column as a string). Co-Authored-By: Claude Opus 5.5 --- tests/test_tallyman_read_csv_contract.py | 94 ++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_tallyman_read_csv_contract.py b/tests/test_tallyman_read_csv_contract.py index 6f8f8e8..6cd84a4 100644 --- a/tests/test_tallyman_read_csv_contract.py +++ b/tests/test_tallyman_read_csv_contract.py @@ -225,3 +225,97 @@ def test_ordinary_reader_kwarg_still_forwarded(project): out = _import(project, "semicsv", p, separator=";") types = _types_of(project, out) assert "a" in types and "b" in types # split into two columns, not one "a;b" + + +# --------------------------------------------------------------------------- # +# #231 — a zone on text with no UTC offset is attached, not converted (ADR-005 D9(a)) +# --------------------------------------------------------------------------- # +NY = "America/New_York" + + +def _zoned_snapshot(project: str, name: str, lines: list[str], dtype: str = f"timestamp({NY!r})"): + """Import a one-column CSV of *lines* under ``{"ts": dtype}`` and return the snapshot's ``ts`` column. + + Read back with pyarrow, straight from the file the import wrote, so the test sees the stored instants and the + stored type with nothing between them and the assertion. + """ + import pyarrow.parquet as pq + + from tallyman_xorq.materialize import snapshot_path + + p = data_dir(project) / f"{name}.csv" + p.write_text("ts\n" + "\n".join(lines) + "\n") + out = _import(project, name, p, {"ts": dtype}) + return pq.read_table(snapshot_path(project, out["hash"])).column("ts") + + +def test_a_zone_attaches_to_offsetless_text(project): + """#231: ``09:30`` under America/New_York is 09:30 there, in winter and in summer. + + polars' reader parsed it as UTC and converted, so it came out as 04:30-05:00 and 05:30-04:00. An empty field + stays null. + """ + ts = _zoned_snapshot(project, "tz_wall", ["2024-01-02 09:30:00", "", "2024-07-02 09:30:00"]) + assert str(ts.type) == f"timestamp[us, tz={NY}]" + assert [v and v.isoformat() for v in ts.to_pylist()] == [ + "2024-01-02T09:30:00-05:00", + None, + "2024-07-02T09:30:00-04:00", + ] + + +def test_a_zoned_nanosecond_column_keeps_its_digits(project): + """#231 with #145: a zone attached to offset-less text keeps the declared ``ns`` unit and every digit.""" + import pyarrow as pa + + ts = _zoned_snapshot(project, "tz_ns", ["2024-01-02 09:30:00.123456789"], f"timestamp({NY!r}, 9)") + assert str(ts.type) == f"timestamp[ns, tz={NY}]" + # 09:30:00.123456789 in New York in January is 14:30:00.123456789 UTC. + assert ts.cast(pa.int64()).to_pylist() == [1_704_205_800_123_456_789] + + +def test_a_wall_clock_time_the_zone_skips_raises(project): + """#231: 02:30 on 2024-03-10 never happens in New York, so no instant is right for it and the import raises. + + polars' reader read it as UTC and stored 2024-03-09 21:30-05:00. + """ + lines = ["2024-03-09 12:00:00", "2024-03-10 01:59:59", "2024-03-10 02:30:00", "2024-03-10 03:00:00"] + with pytest.raises(ValueError) as exc: + _zoned_snapshot(project, "tz_gap", lines) + err = str(exc.value) + assert "'ts'" in err + assert "'2024-03-10 02:30:00' (row 3)" in err + assert f"does not exist in {NY}" in err + + +def test_a_wall_clock_time_the_zone_repeats_raises(project): + """#231: 01:30 on 2024-11-03 happens twice in New York, once at -04:00 and once at -05:00, so the import raises + rather than picking one.""" + with pytest.raises(ValueError) as exc: + _zoned_snapshot(project, "tz_overlap", ["2024-11-02 12:00:00", "2024-11-03 01:30:00"]) + err = str(exc.value) + assert "'ts'" in err + assert "'2024-11-03 01:30:00' (row 2)" in err + assert f"occurs twice in {NY}" in err + + +@pytest.mark.parametrize( + ("lines", "offset_row", "offsetless_row"), + [ + (["2024-01-02 09:30:00", "2024-01-02T09:30:00+00:00"], 2, 1), + (["2024-01-02T09:30:00+00:00", "2024-01-02 09:30:00"], 1, 2), + # Far past the first morsel polars parses, so the rule is the column's, not a batch's. + (["2024-01-02 09:30:00"] * 50_000 + ["2024-01-02T09:30:00+00:00"], 50_001, 1), + ], + ids=["offset-second", "offset-first", "offset-late"], +) +def test_a_zoned_column_mixing_offset_and_offsetless_text_raises(project, lines, offset_row, offsetless_row): + """#231: text with an offset is an instant and text without one is a wall-clock time, and one column is read + one way, so a column with both raises an error that names the column and a row of each kind.""" + with pytest.raises(ValueError) as exc: + _zoned_snapshot(project, "tz_mixed", lines) + err = str(exc.value) + assert "'ts'" in err + assert "mixes text with a UTC offset and text without one" in err + assert f"'{lines[offset_row - 1]}' (row {offset_row})" in err + assert f"'{lines[offsetless_row - 1]}' (row {offsetless_row})" in err From 71a93c2d2b320cb08886ffadaf20f8cbf865a3ad Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Thu, 24 Sep 2026 11:20:40 -0400 Subject: [PATCH 2/4] fix(csv): a zone on offset-less CSV text is attached, not converted (#231) polars' CSV reader, given pl.Datetime(unit, time_zone=tz), parses text with no UTC offset as UTC and converts it into the zone. A zoned column is now read as text and parsed with str.to_datetime(time_unit, time_zone=tz), which attaches the zone to offset-less text and converts text that has an offset, as ADR-005 D9(a) says. The declared unit is kept (#145). It raises instead of shifting a value when offset-less text names a time the zone skips or repeats at a DST change, and when a column mixes text with and without an offset: the streaming engine infers one format per column from its first non-null value and parses every later batch with it. polars 1.40.1's streaming parse writes a batch as nulls, without an error, when that first value matches no format (fixed upstream in pola-rs/polars#28986). The first non-null value of each zoned column is parsed eagerly before the read, which raises in exactly that case. After a failed read, _explain_zoned_failure reads the zoned columns again as text and names the column, the row and the value: a mixed column, a skipped or repeated wall-clock time, or text that is not a timestamp. The ladder now also catches InvalidOperationError, which is how str.to_datetime reports text that does not match the column's format. Guard tests, which pass before and after: offset text converts, UTC reads offset-less text as UTC, a naive timestamp stays naive, a zoned column renamed by position beside inferred columns, and text that is not a timestamp raises as the first value or a later one. ADR-005 D9 gets an implementation note: (a) and (b) were both wrong about polars 1.40.1. Co-Authored-By: Claude Opus 5.5 --- plans/ADR-005-intelligent-csv-import.md | 14 ++ src/tallyman_xorq/io.py | 171 ++++++++++++++++++++++- tests/test_tallyman_read_csv_contract.py | 57 ++++++++ 3 files changed, 240 insertions(+), 2 deletions(-) diff --git a/plans/ADR-005-intelligent-csv-import.md b/plans/ADR-005-intelligent-csv-import.md index 0be3f5f..0d011c6 100644 --- a/plans/ADR-005-intelligent-csv-import.md +++ b/plans/ADR-005-intelligent-csv-import.md @@ -240,6 +240,20 @@ converting (wrong only if naive-local data is mislabelled with a tz); (b) `ns` precision is silently truncated to `µs` by `scan_csv` (the `ns` dtype label is kept but sub-µs digits are dropped). Bad values still raise loudly, never null. +**Implementation note (#231, 2026-09-24, polars 1.40.1).** Both (a) and (b) were +wrong about the reader as installed. (a): the reader parses offset-less text as +UTC and converts it, so `09:30` under `timestamp('America/New_York')` was stored as +`04:30-05:00`. A zoned column is now read as text and parsed with +`str.to_datetime(time_unit, time_zone=tz)` (`io._parse_zoned`), which does what (a) +says: text with an offset is converted into the zone, text without one gets the +zone attached. A wall-clock time the zone skips or repeats at a DST change raises, +and so does a column that mixes text with and without an offset (polars infers one +format per column from its first non-null value). That first value is checked +before the read, because polars 1.40.1's streaming parse writes nulls instead of +raising when it is not a timestamp. The error names the column, the row and the +value. (b): `ns` digits survive the reader and the snapshot, for a naive, a UTC and +a zoned column alike. + ## Alternatives considered - **duckdb as the engine** (sniffer + reject tables for free). Rejected: reverses diff --git a/src/tallyman_xorq/io.py b/src/tallyman_xorq/io.py index 4f17b31..5a74c85 100644 --- a/src/tallyman_xorq/io.py +++ b/src/tallyman_xorq/io.py @@ -122,6 +122,10 @@ def _polars_dtype(dtype): schema's intent). ``decimal`` reads as ``float64`` for now — exact-decimal parsing is deferred to a future UI-driven buckaroo autoclean step (#150). An unmapped type still raises loudly, so a schema mistake fails fast. + + A timestamp with a zone is the type the column ends up with, and is not what + the reader is given: ``_materialize_ordered`` reads that column as text and + ``_parse_zoned`` parses it (#231). """ import polars as pl @@ -376,6 +380,156 @@ def _suggest_schema_dsl(src: Path, scan_kwargs: dict, reserved: tuple[str, ...] return repr(pairs) +def _zoned_columns(overrides: dict) -> dict: + """The pinned columns whose timestamp type carries a zone, as ``{header_name: pl.Datetime}``.""" + import polars as pl + + return {name: dt for name, dt in overrides.items() if isinstance(dt, pl.Datetime) and dt.time_zone is not None} + + +def _parse_zoned(lf, zoned: dict): + """Parse each zoned column of *lf*, which the scan read as text, into its declared type (ADR-005 D9(a), #231). + + polars' CSV reader, given a ``Datetime`` with a zone, parses text with no UTC offset as UTC and converts it into + the zone, so ``09:30`` under America/New_York was stored as ``04:30-05:00``. ``str.to_datetime`` with + ``time_zone`` reads it the way D9(a) says: text with an offset is an instant and is converted into the zone, and + text without one is a wall-clock time in the zone, which gets the zone attached. The declared unit is kept, so + ``ns`` digits survive (#145). + + It raises rather than shift a value: + + - offset-less text that names a time the zone skips or repeats at a daylight-saving change, since no instant, or + two, fit it (``ambiguous="raise"``; a skipped time raises by default); + - a column that mixes text with and without an offset. The streaming engine infers one format per column, from + its first non-null value in file order, and parses every later batch with it (polars' ``strptime-infer`` + node), so text of the other kind fails wherever it first appears; + - text that is not a timestamp, once the format is known. + + The one gap is before the format is known: polars 1.40.1 turns a batch into nulls, without an error, when the + first non-null value it sees matches no format (fixed upstream in pola-rs/polars#28986). The file's first + non-null value is that value, so ``_unreadable_first_zoned_value`` checks it before the read. + """ + import polars as pl + + return lf.with_columns( + pl.col(name).str.to_datetime(time_unit=dt.time_unit, time_zone=dt.time_zone, ambiguous="raise") + for name, dt in zoned.items() + ) + + +# Text ending in a UTC offset after a time of day: Z, +hh, +hhmm or +hh:mm. polars does the parsing; this only sorts +# rows into the two kinds, to explain a failed read of a zoned column. +_UTC_OFFSET = r"\d:\d{2}(?::\d{2}(?:[.,]\d+)?)?\s*(?:[Zz]|[+-]\d{2}(?::?\d{2})?)$" + + +def _zoned_text(src: Path, scan_kwargs: dict, name: str): + """The non-null text of column *name* of *src*, with its row number counted from 1 after the header.""" + import polars as pl + + return ( + pl.scan_csv(str(src), infer_schema_length=0, **scan_kwargs) + .with_row_index("row", offset=1) + .select("row", pl.col(name).alias("text")) + .filter(pl.col("text").is_not_null()) + ) + + +def _cell(found) -> str: + """The value and row number of the one-row frame *found*, for an error message.""" + return f"{found['text'][0]!r} (row {found['row'][0]})" + + +def _zoned_label(name: str, dt, rename: dict) -> str: + """How an error names a zoned column: by its schema name, and its header when a positional schema renamed it.""" + column = repr(name) if rename.get(name, name) == name else f"{rename[name]!r} (header {name!r})" + return f"tallyman_read_csv: column {column} is a timestamp with the zone {dt.time_zone!r}" + + +def _not_a_timestamp(label: str, found) -> str: + return f"{label}, and {_cell(found)} is not a timestamp. Correct the value, or declare the column as a string." + + +# The two ways offset-less text can name no single instant in a zone, as the error words them. +_SKIPPED = "does not exist in {tz}: the clocks skip it at a daylight-saving change, so no instant matches it" +_REPEATED = ( + "occurs twice in {tz}: the clocks pass it twice at a daylight-saving change, once at each offset, so it names " + "two instants" +) + + +def _unreadable_first_zoned_value(src: Path, scan_kwargs: dict, zoned: dict, rename: dict) -> str | None: + """An error message when a zoned column's first non-null value is not a timestamp, else None. + + ``_parse_zoned`` infers the column's format from that value, and polars 1.40.1 writes nulls instead of raising + when it matches no format. An eager parse of the one value raises in exactly that case. Reads the file only up to + the first non-null value of each zoned column. + """ + import polars as pl + + for name, dt in zoned.items(): + first = _zoned_text(src, scan_kwargs, name).head(1).collect() + if first.is_empty(): + continue + try: + first["text"].str.to_datetime(time_unit=dt.time_unit) + except pl.exceptions.PolarsError: + return _not_a_timestamp(_zoned_label(name, dt, rename), first) + return None + + +def _explain_zoned_failure(src: Path, scan_kwargs: dict, zoned: dict, rename: dict) -> str | None: + """Why the zoned columns of *src* did not parse, as an error message, or None when they are not the reason. + + Runs only after a read has failed. It reads each zoned column again as text and looks, in file order, for the + failures of ``_parse_zoned``: a row whose text has a UTC offset when the column's first value has none (or the + other way round), offset-less text that names a time the zone skips or repeats, and text that is not a timestamp. + """ + import polars as pl + + for name, dt in zoned.items(): + tz, label = dt.time_zone, _zoned_label(name, dt, rename) + text = _zoned_text(src, scan_kwargs, name).with_columns(offset=pl.col("text").str.contains(_UTC_OFFSET)) + try: + first = text.head(1).collect() + if first.is_empty(): + continue + other = text.filter(pl.col("offset") != first["offset"][0]).head(1).collect() + if not other.is_empty(): + with_offset, without = (first, other) if first["offset"][0] else (other, first) + return ( + f"{label} and mixes text with a UTC offset and text without one: {_cell(with_offset)} has an " + f"offset and {_cell(without)} does not. Text with an offset is an instant, converted into the " + "zone; text without one is a wall-clock time in the zone. A column is read one way, so write " + "every value with its offset, or none of them." + ) + if not first["offset"][0]: + wall = pl.col("text").str.to_datetime(time_unit=dt.time_unit, strict=False) + earliest = wall.dt.replace_time_zone(tz, ambiguous="earliest", non_existent="null") + unique = wall.dt.replace_time_zone(tz, ambiguous="null", non_existent="null") + found = ( + text.with_columns( + skipped=wall.is_not_null() & earliest.is_null(), + repeated=earliest.is_not_null() & unique.is_null(), + ) + .filter(pl.col("skipped") | pl.col("repeated")) + .head(1) + .collect() + ) + if not found.is_empty(): + why = (_SKIPPED if found["skipped"][0] else _REPEATED).format(tz=tz) + return ( + f"{label}, where text with no UTC offset is a wall-clock time, and {_cell(found)} {why}. " + "tallyman does not pick an instant for it. Correct the value, or write it with its UTC offset." + ) + parsed = pl.col("text").str.to_datetime(time_unit=dt.time_unit, time_zone=tz, strict=False) + found = text.filter(parsed.is_null()).head(1).collect() + except pl.exceptions.PolarsError: + continue # the text defeats this explanation too, and the caller's general error stands + if not found.is_empty(): + return _not_a_timestamp(label, found) + return None + + def _materialize_ordered(src: Path, schema, scan_kwargs: dict, tmp_path: Path, *, write) -> None: """Read *src* (a CSV) into a row-order-stable parquet at *tmp_path* (#143). @@ -418,13 +572,22 @@ def _materialize_ordered(src: Path, schema, scan_kwargs: dict, tmp_path: Path, * f"schema output column name. Rename that column to something other than {ROW_ORDER!r} in the schema." ) explicit_only = plan["all_pinned"] + # A zoned column is read as text and parsed after the scan: the reader itself would convert offset-less text + # from UTC (#231). + zoned = _zoned_columns(plan["overrides"]) if plan else {} + if zoned: + unreadable = _unreadable_first_zoned_value(src, scan_kwargs, zoned, plan["rename"]) + if unreadable is not None: + raise ValueError(unreadable) def _ordered(infer_len): if schema is None: lf = pl.scan_csv(str(src), infer_schema_length=infer_len, **scan_kwargs) else: il = 0 if plan["all_pinned"] else infer_len - lf = pl.scan_csv(str(src), schema_overrides=plan["overrides"], infer_schema_length=il, **scan_kwargs) + overrides = {**plan["overrides"], **dict.fromkeys(zoned, pl.String)} + lf = pl.scan_csv(str(src), schema_overrides=overrides, infer_schema_length=il, **scan_kwargs) + lf = _parse_zoned(lf, zoned) if has_row_order: lf = lf.drop(ROW_ORDER) # overwritten by the fresh index below lf = lf.with_row_index(ROW_ORDER) @@ -443,8 +606,12 @@ def _ordered(infer_len): try: write(_ordered(infer_len), tmp_path) return - except pl.exceptions.ComputeError as exc: + # InvalidOperationError is how str.to_datetime reports text that does not match the column's format. + except (pl.exceptions.ComputeError, pl.exceptions.InvalidOperationError) as exc: last_exc = exc + zone_problem = _explain_zoned_failure(src, scan_kwargs, zoned, plan["rename"]) if zoned else None + if zone_problem is not None: + raise ValueError(zone_problem) from last_exc raise ValueError( f"tallyman_read_csv: the schema does not parse {src.name}: {last_exc}. " f"Suggested schema (whole-file inference): schema={_suggest_schema_dsl(src, scan_kwargs, reserved)}" diff --git a/tests/test_tallyman_read_csv_contract.py b/tests/test_tallyman_read_csv_contract.py index 6cd84a4..5131b96 100644 --- a/tests/test_tallyman_read_csv_contract.py +++ b/tests/test_tallyman_read_csv_contract.py @@ -319,3 +319,60 @@ def test_a_zoned_column_mixing_offset_and_offsetless_text_raises(project, lines, assert "mixes text with a UTC offset and text without one" in err assert f"'{lines[offset_row - 1]}' (row {offset_row})" in err assert f"'{lines[offsetless_row - 1]}' (row {offsetless_row})" in err + + +# These pass before and after the #231 fix: the parse of a zoned column moved out of polars' reader, and they pin what +# it must keep doing. +def test_offset_text_converts_into_the_zone(project): + """#231: text that carries an offset names an instant, which is converted into the zone.""" + ts = _zoned_snapshot(project, "tz_offset", ["2024-01-02T09:30:00+00:00", "2024-07-02T09:30:00Z"]) + assert str(ts.type) == f"timestamp[us, tz={NY}]" + assert [v.isoformat() for v in ts.to_pylist()] == ["2024-01-02T04:30:00-05:00", "2024-07-02T05:30:00-04:00"] + + +def test_utc_reads_offsetless_text_as_utc(project): + """#231: under UTC, attaching the zone and reading as UTC are the same thing. A leading null is skipped when the + column's format is inferred.""" + ts = _zoned_snapshot(project, "tz_utc", ["", "2024-01-02 09:30:00"], "timestamp('UTC')") + assert str(ts.type) == "timestamp[us, tz=UTC]" + assert [v and v.isoformat() for v in ts.to_pylist()] == [None, "2024-01-02T09:30:00+00:00"] + + +def test_a_naive_timestamp_stays_naive(project): + """#231: a timestamp with no zone is read by polars' reader as before, and has no zone.""" + ts = _zoned_snapshot(project, "naive_ts", ["2024-01-02 09:30:00"], "timestamp") + assert str(ts.type) == "timestamp[us]" + assert [v.isoformat() for v in ts.to_pylist()] == ["2024-01-02T09:30:00"] + + +def test_a_zoned_column_renamed_by_position_beside_inferred_columns(project): + """#231: the zoned parse is keyed on the header name and runs before a positional rename, in inference mode.""" + import pyarrow.parquet as pq + + from tallyman_xorq.materialize import snapshot_path + + p = data_dir(project) / "tz_pos.csv" + p.write_text("ts,v\n2024-01-02 09:30:00,1\n") + out = _import(project, "tz_pos", p, (("when", "timestamp('UTC')"), ("&rest", "infer"))) + table = pq.read_table(snapshot_path(project, out["hash"])) + assert [v.isoformat() for v in table.column("when").to_pylist()] == ["2024-01-02T09:30:00+00:00"] + assert table.column("v").to_pylist() == [1] + + +@pytest.mark.parametrize( + "lines", + [["hello", "2024-01-02 09:30:00"], ["2024-01-02 09:30:00", "not a time"]], + ids=["first-value", "later-value"], +) +def test_text_that_is_not_a_timestamp_raises_in_a_zoned_column(project, lines): + """#231: a value that is not a timestamp raises and is never stored as null. + + The first case is the one polars 1.40.1's streaming parse gets wrong on its own: it infers the column's format + from the first value, finds none, and writes the whole batch as nulls, including the good value after it. + """ + bad = next(v for v in lines if not v.startswith("2024")) + with pytest.raises(ValueError) as exc: + _zoned_snapshot(project, "tz_junk", lines) + err = str(exc.value) + assert "'ts'" in err + assert bad in err From 4d38c3ea1d580c2163639df86de857812fdb6614 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Thu, 24 Sep 2026 14:38:47 -0400 Subject: [PATCH 3/4] test: a zoned column imports beside a `row` header and under kept empty strings (#231) Found in review of #244. Reading a zoned column as text broke two CSVs that imported before it: - a header named `row` collides with the row index the zoned-column checks add, and the import fails with a raw polars DuplicateError; - with `missing_utf8_is_empty_string=True`, an empty zoned cell reaches the parse as "" instead of null, and the import fails with "'' is not a timestamp". Co-Authored-By: Claude Opus 5.5 --- tests/test_tallyman_read_csv_contract.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_tallyman_read_csv_contract.py b/tests/test_tallyman_read_csv_contract.py index 5131b96..018dff8 100644 --- a/tests/test_tallyman_read_csv_contract.py +++ b/tests/test_tallyman_read_csv_contract.py @@ -376,3 +376,50 @@ def test_text_that_is_not_a_timestamp_raises_in_a_zoned_column(project, lines): err = str(exc.value) assert "'ts'" in err assert bad in err + + +# Found in review of #244: reading a zoned column as text must not change which CSVs import. +def _csv_snapshot(project: str, name: str, text: str, schema, **reader_options): + """Import *text* as a CSV under *schema* and return the snapshot the import wrote, read with pyarrow.""" + import pyarrow.parquet as pq + + from tallyman_xorq.materialize import snapshot_path + + p = data_dir(project) / f"{name}.csv" + p.write_text(text) + out = _import(project, name, p, schema, **reader_options) + return pq.read_table(snapshot_path(project, out["hash"])) + + +ROW_AND_ZONED = {"row": "int64", "ts": f"timestamp({NY!r})"} + + +def test_a_zoned_column_beside_a_column_named_row(project): + """A header named ``row`` is the file's own column, and the zoned column beside it imports.""" + table = _csv_snapshot(project, "tz_row", "row,ts\n7,2024-01-02 09:30:00\n", ROW_AND_ZONED) + assert table.column("row").to_pylist() == [7] + assert [v.isoformat() for v in table.column("ts").to_pylist()] == ["2024-01-02T09:30:00-05:00"] + + +def test_a_zoned_failure_beside_a_column_named_row_is_explained(project): + """The explanation of a failed zoned column reads the file again; a ``row`` header does not defeat it.""" + with pytest.raises(ValueError) as exc: + _csv_snapshot(project, "tz_row_gap", "row,ts\n1,2024-03-10 01:59:59\n2,2024-03-10 02:30:00\n", ROW_AND_ZONED) + assert f"'2024-03-10 02:30:00' (row 2) does not exist in {NY}" in str(exc.value) + + +@pytest.mark.parametrize( + ("text", "ts", "s"), + [ + ("ts,s\n,a\n2024-01-02 09:30:00,\n", [None, "2024-01-02T09:30:00-05:00"], ["a", ""]), + ("ts,s\n2024-01-02 09:30:00,\n,a\n", ["2024-01-02T09:30:00-05:00", None], ["", "a"]), + ], + ids=["empty-first", "empty-later"], +) +def test_an_empty_zoned_cell_is_null_when_empty_strings_are_kept(project, text, ts, s): + """``missing_utf8_is_empty_string`` keeps an empty cell as ``""`` in a string column. A zoned column is read as + text, but an empty cell in it is still null, as it was when polars' reader parsed it.""" + schema = {"ts": f"timestamp({NY!r})", "s": "string"} + table = _csv_snapshot(project, "tz_empty", text, schema, missing_utf8_is_empty_string=True) + assert [v and v.isoformat() for v in table.column("ts").to_pylist()] == ts + assert table.column("s").to_pylist() == s From e061241457518c7b7301cef6885825ee726a1b8a Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Thu, 24 Sep 2026 14:45:10 -0400 Subject: [PATCH 4/4] fix(csv): a `row` header and kept empty strings no longer break a zoned import (#231) Found in review of #244. - `_zoned_text` adds its row index after selecting the one column it reads, so a column of the file's own named `row` no longer collides with it (a raw polars DuplicateError before). - `_zoned_text_value` turns an empty cell of a zoned column into null before the parse and the checks. `missing_utf8_is_empty_string` is meant for string columns, and a zoned column is read as a string now, so without this an empty cell became "" and failed as "not a timestamp". Co-Authored-By: Claude Opus 5.5 --- src/tallyman_xorq/io.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/tallyman_xorq/io.py b/src/tallyman_xorq/io.py index 5a74c85..51f322d 100644 --- a/src/tallyman_xorq/io.py +++ b/src/tallyman_xorq/io.py @@ -387,6 +387,17 @@ def _zoned_columns(overrides: dict) -> dict: return {name: dt for name, dt in overrides.items() if isinstance(dt, pl.Datetime) and dt.time_zone is not None} +def _zoned_text_value(name: str): + """Zoned column *name*, which the scan read as text, with an empty cell as null. + + ``missing_utf8_is_empty_string`` reads an empty cell of a string column as ``""``. It is meant for the file's + string columns, and an empty cell of a zoned column stays null, as it was when polars' reader parsed the column. + """ + import polars as pl + + return pl.when(pl.col(name) != "").then(pl.col(name)) + + def _parse_zoned(lf, zoned: dict): """Parse each zoned column of *lf*, which the scan read as text, into its declared type (ADR-005 D9(a), #231). @@ -409,10 +420,8 @@ def _parse_zoned(lf, zoned: dict): first non-null value it sees matches no format (fixed upstream in pola-rs/polars#28986). The file's first non-null value is that value, so ``_unreadable_first_zoned_value`` checks it before the read. """ - import polars as pl - return lf.with_columns( - pl.col(name).str.to_datetime(time_unit=dt.time_unit, time_zone=dt.time_zone, ambiguous="raise") + _zoned_text_value(name).str.to_datetime(time_unit=dt.time_unit, time_zone=dt.time_zone, ambiguous="raise") for name, dt in zoned.items() ) @@ -423,13 +432,16 @@ def _parse_zoned(lf, zoned: dict): def _zoned_text(src: Path, scan_kwargs: dict, name: str): - """The non-null text of column *name* of *src*, with its row number counted from 1 after the header.""" + """The non-null text of column *name* of *src*, with its row number counted from 1 after the header. + + The row index is added after the select, so a column of the file's own named ``row`` does not collide with it. + """ import polars as pl return ( pl.scan_csv(str(src), infer_schema_length=0, **scan_kwargs) + .select(_zoned_text_value(name).alias("text")) .with_row_index("row", offset=1) - .select("row", pl.col(name).alias("text")) .filter(pl.col("text").is_not_null()) )