parallel csv scan (#6801)

* parallel csv scan

* add max line length

* Update according to review comments

* Update Configuration doc

---------

Co-authored-by: Andrew Lamb <[email protected]>

Cargo.toml

@@ -17,40 +17,26 @@
 
 [workspace]
 exclude = ["datafusion-cli"]
-members = [
-    "datafusion/common",
-    "datafusion/core",
-    "datafusion/expr",
-    "datafusion/execution",
-    "datafusion/optimizer",
-    "datafusion/physical-expr",
-    "datafusion/proto",
-    "datafusion/proto/gen",
-    "datafusion/row",
-    "datafusion/sql",
-    "datafusion/substrait",
-    "datafusion-examples",
-    "test-utils",
-    "benchmarks",
+members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/proto", "datafusion/proto/gen", "datafusion/row", "datafusion/sql", "datafusion/substrait", "datafusion-examples", "test-utils", "benchmarks",
 ]
 resolver = "2"
 
 [workspace.package]
-version = "27.0.0"
-edition = "2021"
-readme = "README.md"
 authors = ["Apache Arrow <[email protected]>"]
-license = "Apache-2.0"
+edition = "2021"
 homepage = "https://github.com/apache/arrow-datafusion"
+license = "Apache-2.0"
+readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.64"
+version = "27.0.0"
 
 [workspace.dependencies]
 arrow = { version = "43.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
-arrow-flight = { version = "43.0.0", features = ["flight-sql-experimental"] }
+arrow-array = { version = "43.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = { version = "43.0.0", default-features = false }
+arrow-flight = { version = "43.0.0", features = ["flight-sql-experimental"] }
 arrow-schema = { version = "43.0.0", default-features = false }
-arrow-array = { version = "43.0.0", default-features = false, features = ["chrono-tz"] }
 parquet = { version = "43.0.0", features = ["arrow", "async", "object_store"] }
 sqlparser = { version = "0.35", features = ["visitor"] }
 

datafusion/common/src/config.rs

@@ -323,10 +323,13 @@ config_namespace! {
         /// long runner execution, all types of joins may encounter out-of-memory errors.
         pub allow_symmetric_joins_without_pruning: bool, default = true
 
-        /// When set to true, file groups will be repartitioned to achieve maximum parallelism.
-        /// Currently supported only for Parquet format in which case
-        /// multiple row groups from the same file may be read concurrently. If false then each
-        /// row group is read serially, though different files may be read in parallel.
+        /// When set to `true`, file groups will be repartitioned to achieve maximum parallelism.
+        /// Currently Parquet and CSV formats are supported.
+        ///
+        /// If set to `true`, all files will be repartitioned evenly (i.e., a single large file
+        /// might be partitioned into smaller chunks) for parallel scanning.
+        /// If set to `false`, different files will be read in parallel, but repartitioning won't
+        /// happen within a single file.
         pub repartition_file_scans: bool, default = true
 
         /// Should DataFusion repartition data using the partitions keys to execute window

datafusion/core/Cargo.toml

@@ -107,6 +107,7 @@ env_logger = "0.10"
 half = "2.2.1"
 postgres-protocol = "0.6.4"
 postgres-types = { version = "0.2.4", features = ["derive", "with-chrono-0_4"] }
+regex = "1.5.4"
 rstest = "0.18.0"
 rust_decimal = { version = "1.27.0", features = ["tokio-pg"] }
 sqllogictest = "0.14.0"