diff --git a/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py b/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py index 0561369a7..fffc7f29a 100644 --- a/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py +++ b/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py @@ -28,6 +28,8 @@ logger = logging.getLogger("airbyte") +CSV_FIELD_MAX_BYTES = 2**31 + @dataclass class GzipParser(Parser): @@ -176,6 +178,7 @@ def parse(self, data: BufferedIOBase) -> PARSER_OUTPUT_TYPE: Parse CSV data from decompressed bytes. """ text_data = TextIOWrapper(data, encoding=self.encoding) # type: ignore + csv.field_size_limit(CSV_FIELD_MAX_BYTES) reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",") for row in reader: if self.set_values_to_none: diff --git a/unit_tests/sources/declarative/decoders/test_composite_decoder.py b/unit_tests/sources/declarative/decoders/test_composite_decoder.py index 8af9a4b4c..6947b1eba 100644 --- a/unit_tests/sources/declarative/decoders/test_composite_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_composite_decoder.py @@ -268,6 +268,24 @@ def test_composite_raw_decoder_csv_parser_values(requests_mock, encoding: str, d assert parsed_records == expected_data +def test_composite_raw_decoder_csv_parser_supports_large_field_values(requests_mock): + large_value = "a" * (200 * 1024) + requests_mock.register_uri( + "GET", + "https://airbyte.io/", + content=f"value\n{large_value}\n".encode(), + ) + response = requests.get("https://airbyte.io/", stream=True) + previous_limit = csv.field_size_limit(131_072) + + try: + parsed_records = list(CompositeRawDecoder(parser=CsvParser()).decode(response)) + finally: + csv.field_size_limit(previous_limit) + + assert parsed_records == [{"value": large_value}] + + @pytest.mark.parametrize("set_values_to_none", [None, [""], ["--"]]) def test_composite_raw_decoder_parse_empty_strings( requests_mock, set_values_to_none: List[str] | None