Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cpp/src/io/orc/stripe_data.cu
Original file line number Diff line number Diff line change
Expand Up @@ -1211,8 +1211,9 @@ __global__ void __launch_bounds__(block_size)
uint32_t skippedrows = min(static_cast<uint32_t>(first_row - row_in), nrows);
uint32_t skip_count = 0;
for (uint32_t i = t * 32; i < skippedrows; i += 32 * 32) {
uint32_t bits = s->vals.u32[i >> 5];
if (i + 32 > skippedrows) { bits &= (1 << (skippedrows - i)) - 1; }
Comment thread
vuule marked this conversation as resolved.
uint32_t bits = (i + 32 <= skippedrows)
? s->vals.u32[i >> 5]
: (rle8_read_bool32(s->vals.u32, i) & ((1 << (skippedrows - i)) - 1));
skip_count += __popc(bits);
}
skip_count = warp_reduce(temp_storage[t / 32]).Sum(skip_count);
Expand Down
25 changes: 25 additions & 0 deletions python/cudf/cudf/tests/test_orc.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import pandas as pd
import pyarrow as pa
import pyarrow.orc
import pyorc

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CI fails because of this import

ModuleNotFoundError: No module named 'pyorc'

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Created a PR for it rapidsai/integration#215

import pytest

import cudf
Expand Down Expand Up @@ -318,6 +319,30 @@ def test_orc_read_rows(datadir, skiprows, num_rows):
np.testing.assert_allclose(pdf, gdf)


def test_orc_read_skiprows(tmpdir):
buff = BytesIO()
df = pd.DataFrame(
{"a": [1, 0, 1, 0, None, 1, 1, 1, 0, None, 0, 0, 1, 1, 1, 1]},
dtype=pd.BooleanDtype(),
)
writer = pyorc.Writer(buff, pyorc.Struct(a=pyorc.Boolean()))
tuples = list(
map(
lambda x: (None,) if x[0] is pd.NA else x,
list(df.itertuples(index=False, name=None)),
)
)
writer.writerows(tuples)
writer.close()

skiprows = 10

expected = cudf.read_orc(buff)[skiprows::].reset_index(drop=True)
got = cudf.read_orc(buff, skiprows=skiprows)

assert_eq(expected, got)


def test_orc_reader_uncompressed_block(datadir):
path = datadir / "uncompressed_snappy.orc"
try:
Expand Down