Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support unprefixed config format options #9594

Closed
wants to merge 2 commits into from
Closed

support unprefixed config format options #9594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7af17ba
support unprefixed config format options
tinfoil-knight Mar 13, 2024
2e9f919
avoid clones, stop modifying any namespaced key
tinfoil-knight Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions datafusion/sql/src/statement.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -855,6 +855,31 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
let file_type = try_infer_file_type(&mut options, &statement.target)?;
let partition_by = take_partition_by(&mut options);

let rename_keys = |options: HashMap<String, String>| {
options
.into_iter()
.map(|(k, v)| {
// If config does not belong to any namespace, assume it is
// a legacy option and apply file_type namespace for backwards
// compatibility.
if !k.contains('.') {
let new_key = format!("{}.{}", file_type, k);
(new_key, v)
} else {
(k, v)
}
})
.collect()
};

// Renames un-prefixed keys to support legacy format specific options
let options = match &file_type {
FileType::CSV | FileType::JSON => rename_keys(options),
#[cfg(feature = "parquet")]
FileType::PARQUET => rename_keys(options),
tinfoil-knight marked this conversation as resolved.
Show resolved Hide resolved
_ => options,
};

Ok(LogicalPlan::Copy(CopyTo {
input: Arc::new(input),
output_url: statement.target,
Expand Down
32 changes: 32 additions & 0 deletions datafusion/sqllogictest/test_files/copy.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,3 +484,35 @@ COPY (select col2, sum(col1) from source_table
# Copy from table with non literal
query error DataFusion error: SQL error: ParserError\("Expected ',' or '\)' after option definition, found: \+"\)
COPY source_table to '/tmp/table.parquet' (row_group_size 55 + 102);


# Legacy Format Options Support
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️


# Copy with legacy format options for Parquet
query IT
COPY source_table TO 'test_files/scratch/copy/format_table/' (
format parquet,
compression snappy,
'compression::col1' 'zstd(5)'
);
----
2

# Copy with legacy format options for JSON
query IT
COPY source_table to 'test_files/scratch/copy/format_table' (format json, compression gzip);
----
2

# Copy with legacy format options for CSV
query IT
COPY source_table to 'test_files/scratch/copy/format_table' (
format csv,
has_header false,
compression xz,
datetime_format '%FT%H:%M:%S.%9f',
delimiter ';',
null_value 'NULLVAL'
);
----
2
Loading