Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions rust/arrow/src/csv/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,32 @@ fn infer_field_schema(string: &str) -> DataType {
///
/// If `max_read_records` is not set, the whole file is read to infer its schema.
///
/// Return infered schema and number of records used for inference. This function does not change
/// reader cursor offset.
pub fn infer_file_schema<R: Read + Seek>(
reader: &mut R,
delimiter: u8,
max_read_records: Option<usize>,
has_header: bool,
) -> Result<(Schema, usize)> {
let saved_offset = reader.seek(SeekFrom::Current(0))?;

let (schema, records_count) =
infer_reader_schema(reader, delimiter, max_read_records, has_header)?;

// return the reader seek back to the start
reader.seek(SeekFrom::Start(saved_offset))?;

Ok((schema, records_count))
}

/// Infer schema of CSV records provided by struct that implements `Read` trait.
///
/// `max_read_records` controlling the maximum number of records to read. If `max_read_records` is
/// not set, all records are read to infer the schema.
///
/// Return infered schema and number of records used for inference.
fn infer_file_schema<R: Read + Seek>(
pub fn infer_reader_schema<R: Read>(
reader: &mut R,
delimiter: u8,
max_read_records: Option<usize>,
Expand All @@ -121,18 +145,12 @@ fn infer_file_schema<R: Read + Seek>(
.collect()
};

// save the csv reader position after reading headers
let position = csv_reader.position().clone();

let header_length = headers.len();
// keep track of inferred field types
let mut column_types: Vec<HashSet<DataType>> = vec![HashSet::new(); header_length];
// keep track of columns with nulls
let mut nulls: Vec<bool> = vec![false; header_length];

// return csv reader position to after headers
csv_reader.seek(position)?;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a no op.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is a good catch.


let mut records_count = 0;
let mut fields = vec![];

Expand Down Expand Up @@ -184,9 +202,6 @@ fn infer_file_schema<R: Read + Seek>(
}
}

// return the reader seek back to the start
csv_reader.into_inner().seek(SeekFrom::Start(0))?;

Ok((Schema::new(fields), records_count))
}

Expand Down