Skip to content

fix(declarative): raise CSV field size limit in CsvParser #1114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
devin-ai-integration wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix(declarative): raise CSV field size limit in CsvParser #1114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@

logger = logging.getLogger("airbyte")

CSV_FIELD_MAX_BYTES = 2**31


@dataclass
class GzipParser(Parser):
Expand Down Expand Up @@ -176,6 +178,7 @@ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE:
Parse CSV data from decompressed bytes.
"""
text_data = TextIOWrapper(data, encoding=self.encoding) # type: ignore
csv.field_size_limit(CSV_FIELD_MAX_BYTES)
reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
for row in reader:
if self.set_values_to_none:
Expand Down
18 changes: 18 additions & 0 deletions unit_tests/sources/declarative/decoders/test_composite_decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,24 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d
assert parsed_records == expected_data


def test_composite_raw_decoder_csv_parser_supports_large_field_values(requests_mock):
large_value = "a" * (200 * 1024)
requests_mock.register_uri(
"GET",
"https://airbyte.io/",
content=f"value\n{large_value}\n".encode(),
)
response = requests.get("https://airbyte.io/", stream=True)
previous_limit = csv.field_size_limit(131_072)

try:
parsed_records = list(CompositeRawDecoder(parser=CsvParser()).decode(response))
finally:
csv.field_size_limit(previous_limit)

assert parsed_records == [{"value": large_value}]


@pytest.mark.parametrize("set_values_to_none", [None, [""], ["--"]])
def test_composite_raw_decoder_parse_empty_strings(
requests_mock, set_values_to_none: List[str] | None
Expand Down
Loading