Update vendored DuckDB sources to 00cec88

duckdblabs-bot · duckdblabs-bot · commit f7ae8cb492d5 · 2024-09-13T00:31:50.000Z
diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp
diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp
@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "1-dev64"
+#define DUCKDB_PATCH_VERSION "1-dev100"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 1
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.1.1-dev64"
+#define DUCKDB_VERSION "v1.1.1-dev100"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e2b177b759"
+#define DUCKDB_SOURCE_ID "9af117f0e6"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
diff --git a/src/duckdb/src/include/duckdb.h b/src/duckdb/src/include/duckdb.h
@@ -866,8 +866,8 @@ Returns the number of rows present in the result object.
 * @return The number of rows present in the result object.
 */
 DUCKDB_API idx_t duckdb_row_count(duckdb_result *result);
-#endif
 
+#endif
 /*!
 Returns the number of rows changed by the query stored in the result. This is relevant only for INSERT/UPDATE/DELETE
 queries. For other queries the rows_changed will be 0.
@@ -898,9 +898,7 @@ printf("Data for row %d: %d\n", row, data[row]);
 * @return The column data of the specified column.
 */
 DUCKDB_API void *duckdb_column_data(duckdb_result *result, idx_t col);
-#endif
 
-#ifndef DUCKDB_API_NO_DEPRECATED
 /*!
 **DEPRECATED**: Prefer using `duckdb_result_get_chunk` instead.
 
@@ -923,8 +921,8 @@ if (nullmask[row]) {
 * @return The nullmask of the specified column.
 */
 DUCKDB_API bool *duckdb_nullmask_data(duckdb_result *result, idx_t col);
-#endif
 
+#endif
 /*!
 Returns the error message contained within the result. The error is only set if `duckdb_query` returns `DuckDBError`.
 
@@ -990,6 +988,7 @@ Returns the number of data chunks present in the result.
 */
 DUCKDB_API idx_t duckdb_result_chunk_count(duckdb_result result);
 
+#endif
 /*!
 Returns the return_type of the given result, or DUCKDB_RETURN_TYPE_INVALID on error
 
@@ -998,7 +997,6 @@ Returns the return_type of the given result, or DUCKDB_RETURN_TYPE_INVALID on er
 */
 DUCKDB_API duckdb_result_type duckdb_result_return_type(duckdb_result result);
 
-#endif
 //===--------------------------------------------------------------------===//
 // Safe Fetch Functions
 //===--------------------------------------------------------------------===//
@@ -1663,8 +1661,8 @@ Note that the result must be freed with `duckdb_destroy_result`.
 */
 DUCKDB_API duckdb_state duckdb_execute_prepared_streaming(duckdb_prepared_statement prepared_statement,
                                                           duckdb_result *out_result);
-#endif
 
+#endif
 //===--------------------------------------------------------------------===//
 // Extract Statements
 //===--------------------------------------------------------------------===//
@@ -1754,8 +1752,8 @@ Note that after calling `duckdb_pending_prepared_streaming`, the pending result
 */
 DUCKDB_API duckdb_state duckdb_pending_prepared_streaming(duckdb_prepared_statement prepared_statement,
                                                           duckdb_pending_result *out_result);
-#endif
 
+#endif
 /*!
 Closes the pending result and de-allocates all memory allocated for the result.
 
@@ -4038,8 +4036,8 @@ It is not known beforehand how many chunks will be returned by this result.
 * @return The resulting data chunk. Returns `NULL` if the result has an error.
 */
 DUCKDB_API duckdb_data_chunk duckdb_stream_fetch_chunk(duckdb_result result);
-#endif
 
+#endif
 /*!
 Fetches a data chunk from a duckdb_result. This function should be called repeatedly until the result is exhausted.
 
diff --git a/src/duckdb/src/include/duckdb/main/config.hpp b/src/duckdb/src/include/duckdb/main/config.hpp
@@ -245,7 +245,7 @@ struct DBConfigOptions {
 	//! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
 	idx_t allocator_flush_threshold = 134217728ULL;
 	//! If bulk deallocation larger than this occurs, flush outstanding allocations (1 << 30, ~1GB)
-	idx_t allocator_bulk_deallocation_flush_threshold = 1073741824ULL;
+	idx_t allocator_bulk_deallocation_flush_threshold = 536870912ULL;
 	//! Whether the allocator background thread is enabled
 	bool allocator_background_threads = false;
 	//! DuckDB API surface
diff --git a/src/duckdb/src/include/duckdb/main/extension.hpp b/src/duckdb/src/include/duckdb/main/extension.hpp
@@ -50,6 +50,7 @@ struct ParsedExtensionMetaData {
 	string duckdb_capi_version;
 	string extension_version;
 	string signature;
+	string extension_abi_metadata;
 
 	bool AppearsValid() {
 		return magic_value == EXPECTED_MAGIC_VALUE;
diff --git a/src/duckdb/src/include/duckdb/main/extension_helper.hpp b/src/duckdb/src/include/duckdb/main/extension_helper.hpp
@@ -124,7 +124,7 @@ class ExtensionHelper {
 
 	static bool CheckExtensionSignature(FileHandle &handle, ParsedExtensionMetaData &parsed_metadata,
 	                                    const bool allow_community_extensions);
-	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata);
+	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata) noexcept;
 	static ParsedExtensionMetaData ParseExtensionMetaData(FileHandle &handle);
 
 	//! Get the extension url template, containing placeholders for version, platform and extension name
diff --git a/src/duckdb/src/include/duckdb/main/settings.hpp b/src/duckdb/src/include/duckdb/main/settings.hpp
@@ -506,7 +506,7 @@ struct HTTPProxyUsername {
 };
 
 struct HTTPProxyPassword {
-	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Name = "http_proxy_password";
 	static constexpr const char *Description = "Password for HTTP proxy";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
 	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
@@ -596,7 +596,9 @@ struct StreamingBufferSize {
 struct MaximumTempDirectorySize {
 	static constexpr const char *Name = "max_temp_directory_size";
 	static constexpr const char *Description =
-	    "The maximum amount of data stored inside the 'temp_directory' (when set) (e.g. 1GB)";
+	    "The maximum amount of data stored inside the 'temp_directory' (when set). If the `temp_directory` is set to "
+	    "an existing directory, this option defaults to the available disk space on "
+	    "that drive. Otherwise, it defaults to 0 (implying that the temporary directory is not used).";
 	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
 	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
 	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
diff --git a/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp b/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp
@@ -41,7 +41,7 @@ class BufferPool {
 	friend class StandardBufferManager;
 
 public:
-	explicit BufferPool(idx_t maximum_memory, bool track_eviction_timestamps);
+	BufferPool(idx_t maximum_memory, bool track_eviction_timestamps, idx_t allocator_bulk_deallocation_flush_threshold);
 	virtual ~BufferPool();
 
 	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
diff --git a/src/duckdb/src/main/database.cpp b/src/duckdb/src/main/database.cpp
@@ -2,29 +2,29 @@
 
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/virtual_file_system.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/execution/operator/helper/physical_set.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/connection_manager.hpp"
+#include "duckdb/main/database_file_opener.hpp"
 #include "duckdb/main/database_manager.hpp"
 #include "duckdb/main/database_path_and_type.hpp"
+#include "duckdb/main/db_instance_cache.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/main/extension_helper.hpp"
 #include "duckdb/main/secret/secret_manager.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/parsed_data/attach_info.hpp"
+#include "duckdb/planner/collation_binding.hpp"
 #include "duckdb/planner/extension_callback.hpp"
 #include "duckdb/storage/object_cache.hpp"
 #include "duckdb/storage/standard_buffer_manager.hpp"
 #include "duckdb/storage/storage_extension.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
-#include "duckdb/execution/index/index_type_set.hpp"
-#include "duckdb/main/database_file_opener.hpp"
-#include "duckdb/planner/collation_binding.hpp"
-#include "duckdb/main/db_instance_cache.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -436,7 +436,8 @@ void DatabaseInstance::Configure(DBConfig &new_config, const char *database_path
 		config.buffer_pool = std::move(new_config.buffer_pool);
 	} else {
 		config.buffer_pool = make_shared_ptr<BufferPool>(config.options.maximum_memory,
-		                                                 config.options.buffer_manager_track_eviction_timestamps);
+		                                                 config.options.buffer_manager_track_eviction_timestamps,
+		                                                 config.options.allocator_bulk_deallocation_flush_threshold);
 	}
 }
 
diff --git a/src/duckdb/src/main/extension.cpp b/src/duckdb/src/main/extension.cpp
@@ -62,7 +62,7 @@ string ParsedExtensionMetaData::GetInvalidMetadataError() {
 			                       DUCKDB_EXTENSION_API_VERSION_MINOR, DUCKDB_EXTENSION_API_VERSION_PATCH);
 		}
 	} else {
-		throw InternalException("Unknown ABI type for extension: " + EnumUtil::ToString(abi_type));
+		throw InternalException("Unknown ABI type for extension: " + extension_abi_metadata);
 	}
 
 	if (engine_platform != platform) {
diff --git a/src/duckdb/src/main/extension/extension_load.cpp b/src/duckdb/src/main/extension/extension_load.cpp
@@ -173,7 +173,7 @@ static string FilterZeroAtEnd(string s) {
 	return s;
 }
 
-ParsedExtensionMetaData ExtensionHelper::ParseExtensionMetaData(const char *metadata) {
+ParsedExtensionMetaData ExtensionHelper::ParseExtensionMetaData(const char *metadata) noexcept {
 	ParsedExtensionMetaData result;
 
 	vector<string> metadata_field;
@@ -194,12 +194,18 @@ ParsedExtensionMetaData ExtensionHelper::ParseExtensionMetaData(const char *meta
 
 	result.extension_version = FilterZeroAtEnd(metadata_field[3]);
 
-	result.abi_type = EnumUtil::FromString<ExtensionABIType>(FilterZeroAtEnd(metadata_field[4]));
+	auto extension_abi_metadata = FilterZeroAtEnd(metadata_field[4]);
 
-	if (result.abi_type == ExtensionABIType::C_STRUCT) {
+	if (extension_abi_metadata == "C_STRUCT") {
+		result.abi_type = ExtensionABIType::C_STRUCT;
 		result.duckdb_capi_version = FilterZeroAtEnd(metadata_field[2]);
-	} else if (result.abi_type == ExtensionABIType::CPP) {
+	} else if (extension_abi_metadata == "CPP" || extension_abi_metadata.empty()) {
+		result.abi_type = ExtensionABIType::CPP;
 		result.duckdb_version = FilterZeroAtEnd(metadata_field[2]);
+	} else {
+		result.abi_type = ExtensionABIType::UNKNOWN;
+		result.duckdb_version = "unknown";
+		result.extension_abi_metadata = extension_abi_metadata;
 	}
 
 	result.signature = string(metadata, ParsedExtensionMetaData::FOOTER_SIZE - ParsedExtensionMetaData::SIGNATURE_SIZE);
diff --git a/src/duckdb/src/optimizer/join_order/relation_manager.cpp b/src/duckdb/src/optimizer/join_order/relation_manager.cpp
@@ -85,6 +85,7 @@ static bool OperatorNeedsRelation(LogicalOperatorType op_type) {
 	case LogicalOperatorType::LOGICAL_PROJECTION:
 	case LogicalOperatorType::LOGICAL_EXPRESSION_GET:
 	case LogicalOperatorType::LOGICAL_GET:
+	case LogicalOperatorType::LOGICAL_UNNEST:
 	case LogicalOperatorType::LOGICAL_DELIM_GET:
 	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY:
 	case LogicalOperatorType::LOGICAL_WINDOW:
@@ -269,6 +270,20 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica
 		AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type);
 		return true;
 	}
+	case LogicalOperatorType::LOGICAL_UNNEST: {
+		// optimize children of unnest
+		RelationStats child_stats;
+		auto child_optimizer = optimizer.CreateChildOptimizer();
+		op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats);
+		// the extracted cardinality should be set for window
+		if (!datasource_filters.empty()) {
+			child_stats.cardinality = LossyNumericCast<idx_t>(static_cast<double>(child_stats.cardinality) *
+			                                                  RelationStatisticsHelper::DEFAULT_SELECTIVITY);
+		}
+		ModifyStatsIfLimit(limit_op.get(), child_stats);
+		AddRelation(input_op, parent, child_stats);
+		return true;
+	}
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		auto &join = op->Cast<LogicalComparisonJoin>();
 		// Adding relations of the left side to the current join order optimizer
diff --git a/src/duckdb/src/planner/binder.cpp b/src/duckdb/src/planner/binder.cpp
@@ -6,6 +6,7 @@
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/subquery_expression.hpp"
 #include "duckdb/parser/parsed_expression_iterator.hpp"
@@ -21,7 +22,6 @@
 #include "duckdb/planner/operator/logical_sample.hpp"
 #include "duckdb/planner/query_node/list.hpp"
 #include "duckdb/planner/tableref/list.hpp"
-#include "duckdb/main/database.hpp"
 
 #include <algorithm>
 
@@ -134,10 +134,8 @@ BoundStatement Binder::BindWithCTE(T &statement) {
 		}
 		MoveCorrelatedExpressions(*tail.child_binder);
 
-		// extract operator below root operation
-		auto plan = std::move(bound_statement.plan->children[0]);
-		bound_statement.plan->children.clear();
-		bound_statement.plan->children.push_back(CreatePlan(*bound_cte, std::move(plan)));
+		auto plan = std::move(bound_statement.plan);
+		bound_statement.plan = CreatePlan(*bound_cte, std::move(plan));
 	} else {
 		bound_statement = Bind(statement.template Cast<T>());
 	}
diff --git a/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp b/src/duckdb/src/planner/binder/statement/bind_copy_database.cpp
@@ -54,7 +54,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 	ExportEntries entries;
 	PhysicalExport::ExtractEntries(context, source_schemas, entries);
 
-	unique_ptr<LogicalOperator> result;
+	vector<unique_ptr<LogicalOperator>> insert_nodes;
 	for (auto &table_ref : entries.tables) {
 		auto &table = table_ref.get().Cast<TableCatalogEntry>();
 		// generate the insert statement
@@ -82,17 +82,10 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		insert_stmt.select_statement = std::move(select_stmt);
 		auto bound_insert = Bind(insert_stmt);
 		auto insert_plan = std::move(bound_insert.plan);
-		if (result) {
-			// use UNION ALL to combine the individual copy statements into a single node
-			auto copy_union =
-			    make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(insert_plan), std::move(result),
-			                                   LogicalOperatorType::LOGICAL_UNION, true, false);
-			result = std::move(copy_union);
-		} else {
-			result = std::move(insert_plan);
-		}
+		insert_nodes.push_back(std::move(insert_plan));
 	}
-	if (!result) {
+	unique_ptr<LogicalOperator> result;
+	if (insert_nodes.empty()) {
 		vector<LogicalType> result_types;
 		result_types.push_back(LogicalType::BIGINT);
 		vector<unique_ptr<Expression>> expression_list;
@@ -101,6 +94,23 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
 		expressions.push_back(std::move(expression_list));
 		result = make_uniq<LogicalExpressionGet>(GenerateTableIndex(), std::move(result_types), std::move(expressions));
 		result->children.push_back(make_uniq<LogicalDummyScan>(GenerateTableIndex()));
+	} else {
+		// use UNION ALL to combine the individual copy statements into a single node
+		while (insert_nodes.size() > 1) {
+			vector<unique_ptr<LogicalOperator>> new_nodes;
+			for (idx_t i = 0; i < insert_nodes.size(); i += 2) {
+				if (i + 1 == insert_nodes.size()) {
+					new_nodes.push_back(std::move(insert_nodes[i]));
+				} else {
+					auto copy_union = make_uniq<LogicalSetOperation>(
+					    GenerateTableIndex(), 1U, std::move(insert_nodes[i]), std::move(insert_nodes[i + 1]),
+					    LogicalOperatorType::LOGICAL_UNION, true, false);
+					new_nodes.push_back(std::move(copy_union));
+				}
+			}
+			insert_nodes = std::move(new_nodes);
+		}
+		result = std::move(insert_nodes[0]);
 	}
 	return result;
 }
diff --git a/src/duckdb/src/storage/buffer/buffer_pool.cpp b/src/duckdb/src/storage/buffer/buffer_pool.cpp
@@ -194,8 +194,11 @@ void EvictionQueue::PurgeIteration(const idx_t purge_size) {
 	total_dead_nodes -= actually_dequeued - alive_nodes;
 }
 
-BufferPool::BufferPool(idx_t maximum_memory, bool track_eviction_timestamps)
-    : maximum_memory(maximum_memory), track_eviction_timestamps(track_eviction_timestamps),
+BufferPool::BufferPool(idx_t maximum_memory, bool track_eviction_timestamps,
+                       idx_t allocator_bulk_deallocation_flush_threshold)
+    : maximum_memory(maximum_memory),
+      allocator_bulk_deallocation_flush_threshold(allocator_bulk_deallocation_flush_threshold),
+      track_eviction_timestamps(track_eviction_timestamps),
       temporary_memory_manager(make_uniq<TemporaryMemoryManager>()) {
 	queues.reserve(FILE_BUFFER_TYPE_COUNT);
 	for (idx_t i = 0; i < FILE_BUFFER_TYPE_COUNT; i++) {
@@ -283,6 +286,9 @@ BufferPool::EvictionResult BufferPool::EvictBlocksInternal(EvictionQueue &queue,
 	bool found = false;
 
 	if (memory_usage.GetUsedMemory(MemoryUsageCaches::NO_FLUSH) <= memory_limit) {
+		if (Allocator::SupportsFlush() && extra_memory > allocator_bulk_deallocation_flush_threshold) {
+			Allocator::FlushAll();
+		}
 		return {true, std::move(r)};
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,7 @@ string ParsedExtensionMetaData::GetInvalidMetadataError() {`
`62`	`62`	`DUCKDB_EXTENSION_API_VERSION_MINOR, DUCKDB_EXTENSION_API_VERSION_PATCH);`
`63`	`63`	`}`
`64`	`64`	`} else {`
`65`		`- throw InternalException("Unknown ABI type for extension: " + EnumUtil::ToString(abi_type));`
	`65`	`+ throw InternalException("Unknown ABI type for extension: " + extension_abi_metadata);`
`66`	`66`	`}`
`67`	`67`
`68`	`68`	`if (engine_platform != platform) {`