diff --git a/velox/connectors/hive/HiveDataSink.cpp b/velox/connectors/hive/HiveDataSink.cpp index d20f144276e..a185ca20e9d 100644 --- a/velox/connectors/hive/HiveDataSink.cpp +++ b/velox/connectors/hive/HiveDataSink.cpp @@ -693,8 +693,7 @@ uint32_t HiveDataSink::appendWriter(const HiveWriterId& id) { // through insertTableHandle) // 2. Otherwise, acquire user defined session properties. // 3. Lastly, acquire general hive connector configs. - options->processSessionConfigs(*connectorSessionProperties); - options->processHiveConnectorConfigs(*hiveConfig_->config()); + options->processConfigs(*hiveConfig_->config(), *connectorSessionProperties); // Only overwrite options in case they were not already provided. if (options->schema == nullptr) { diff --git a/velox/dwio/common/Options.h b/velox/dwio/common/Options.h index a3aaa8ff3b2..a695e30b45b 100644 --- a/velox/dwio/common/Options.h +++ b/velox/dwio/common/Options.h @@ -620,8 +620,9 @@ struct WriterOptions { // WriterOption implementations should provide this function to specify how to // process format-specific session and connector configs. - virtual void processSessionConfigs(const config::ConfigBase&) {} - virtual void processHiveConnectorConfigs(const config::ConfigBase&) {} + virtual void processConfigs( + const config::ConfigBase& connectorConfig, + const config::ConfigBase& session) {}; virtual ~WriterOptions() = default; }; diff --git a/velox/dwio/dwrf/writer/Writer.cpp b/velox/dwio/dwrf/writer/Writer.cpp index 1312ee3b9c4..2c38455f961 100644 --- a/velox/dwio/dwrf/writer/Writer.cpp +++ b/velox/dwio/dwrf/writer/Writer.cpp @@ -32,6 +32,11 @@ using facebook::velox::common::testutil::TestValue; namespace facebook::velox::dwrf { + +void WriterOptions::processConfigs( + const config::ConfigBase& connectorConfig, + const config::ConfigBase& session) {}; + namespace { dwio::common::StripeProgress getStripeProgress(const WriterContext& context) { diff --git a/velox/dwio/dwrf/writer/Writer.h b/velox/dwio/dwrf/writer/Writer.h index 9dbe2252e0b..5056faa3942 100644 --- a/velox/dwio/dwrf/writer/Writer.h +++ b/velox/dwio/dwrf/writer/Writer.h @@ -51,6 +51,10 @@ struct WriterOptions : public dwio::common::WriterOptions { WriterContext& context, const velox::dwio::common::TypeWithId& type)> columnWriterFactory; + + void processConfigs( + const config::ConfigBase& connectorConfig, + const config::ConfigBase& session) override; }; class Writer : public dwio::common::Writer { diff --git a/velox/dwio/parquet/writer/Writer.cpp b/velox/dwio/parquet/writer/Writer.cpp index 0a92aa916e0..c27fd6b7b54 100644 --- a/velox/dwio/parquet/writer/Writer.cpp +++ b/velox/dwio/parquet/writer/Writer.cpp @@ -435,28 +435,22 @@ std::optional getTimestampTimeZone( } // namespace -void WriterOptions::processSessionConfigs(const config::ConfigBase& config) { +void WriterOptions::processConfigs( + const config::ConfigBase& connectorConfig, + const config::ConfigBase& session) { if (!parquetWriteTimestampUnit) { parquetWriteTimestampUnit = - getTimestampUnit(config, kParquetSessionWriteTimestampUnit); + getTimestampUnit(session, kParquetSessionWriteTimestampUnit).has_value() + ? getTimestampUnit(session, kParquetSessionWriteTimestampUnit) + : getTimestampUnit(connectorConfig, kParquetSessionWriteTimestampUnit); } - - if (!parquetWriteTimestampTimeZone) { - parquetWriteTimestampTimeZone = - getTimestampTimeZone(config, core::QueryConfig::kSessionTimezone); - } -} - -void WriterOptions::processHiveConnectorConfigs( - const config::ConfigBase& config) { - if (!parquetWriteTimestampUnit) { - parquetWriteTimestampUnit = - getTimestampUnit(config, kParquetHiveConnectorWriteTimestampUnit); - } - if (!parquetWriteTimestampTimeZone) { parquetWriteTimestampTimeZone = - getTimestampTimeZone(config, core::QueryConfig::kSessionTimezone); + getTimestampTimeZone(session, core::QueryConfig::kSessionTimezone) + .has_value() + ? getTimestampTimeZone(session, core::QueryConfig::kSessionTimezone) + : getTimestampTimeZone( + connectorConfig, core::QueryConfig::kSessionTimezone); } } diff --git a/velox/dwio/parquet/writer/Writer.h b/velox/dwio/parquet/writer/Writer.h index e3da877be4f..2b0940304e0 100644 --- a/velox/dwio/parquet/writer/Writer.h +++ b/velox/dwio/parquet/writer/Writer.h @@ -123,8 +123,9 @@ struct WriterOptions : public dwio::common::WriterOptions { "hive.parquet.writer.timestamp-unit"; // Process hive connector and session configs. - void processSessionConfigs(const config::ConfigBase& config) override; - void processHiveConnectorConfigs(const config::ConfigBase& config) override; + void processConfigs( + const config::ConfigBase& connectorConfig, + const config::ConfigBase& session) override; }; // Writes Velox vectors into a DataSink using Arrow Parquet writer. diff --git a/velox/exec/fuzzer/PrestoQueryRunner.cpp b/velox/exec/fuzzer/PrestoQueryRunner.cpp index 599552b7ab8..05e3e712fcf 100644 --- a/velox/exec/fuzzer/PrestoQueryRunner.cpp +++ b/velox/exec/fuzzer/PrestoQueryRunner.cpp @@ -17,6 +17,7 @@ #include "velox/exec/fuzzer/PrestoQueryRunner.h" #include // @manual #include +#include #include #include "velox/common/base/Fs.h" #include "velox/common/encode/Base64.h" @@ -44,7 +45,8 @@ void writeToFile( memory::MemoryPool* pool) { VELOX_CHECK_GT(data.size(), 0); - auto options = std::make_shared(); + std::shared_ptr options = + std::make_shared(); options->schema = data[0]->type(); options->memoryPool = pool;