Dependency manager powered export

.clang-tidy

@@ -1,4 +1,4 @@
-Checks:          '-*,clang-diagnostic-*,bugprone-*,performance-*,google-explicit-constructor,google-build-using-namespace,google-runtime-int,misc-definitions-in-headers,modernize-use-nullptr,modernize-use-override,-bugprone-macro-parentheses,readability-braces-around-statements,-bugprone-branch-clone,readability-identifier-naming,hicpp-exception-baseclass,misc-throw-by-value-catch-by-reference,-bugprone-signed-char-misuse,-bugprone-misplaced-widening-cast,-bugprone-sizeof-expression,-bugprone-narrowing-conversions,-bugprone-easily-swappable-parameters,google-global-names-in-headers,llvm-header-guard,misc-definitions-in-headers,modernize-use-emplace,modernize-use-bool-literals,-performance-inefficient-string-concatenation,-performance-no-int-to-ptr,readability-container-size-empty,cppcoreguidelines-pro-type-cstyle-cast,-llvm-header-guard,-performance-enum-size'
+Checks:          '-*,clang-diagnostic-*,bugprone-*,performance-*,google-explicit-constructor,google-build-using-namespace,google-runtime-int,misc-definitions-in-headers,modernize-use-nullptr,modernize-use-override,-bugprone-macro-parentheses,readability-braces-around-statements,-bugprone-branch-clone,readability-identifier-naming,hicpp-exception-baseclass,misc-throw-by-value-catch-by-reference,-bugprone-signed-char-misuse,-bugprone-misplaced-widening-cast,-bugprone-sizeof-expression,-bugprone-narrowing-conversions,-bugprone-easily-swappable-parameters,google-global-names-in-headers,llvm-header-guard,misc-definitions-in-headers,modernize-use-emplace,modernize-use-bool-literals,-performance-inefficient-string-concatenation,-performance-no-int-to-ptr,readability-container-size-empty,cppcoreguidelines-pro-type-cstyle-cast,-llvm-header-guard,-performance-enum-size,cppcoreguidelines-pro-type-const-cast,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-interfaces-global-init,cppcoreguidelines-slicing,cppcoreguidelines-rvalue-reference-param-not-moved,cppcoreguidelines-virtual-class-destructor'
 WarningsAsErrors: '*'
 HeaderFilterRegex: 'src/include/duckdb/.*'
 FormatStyle:     none
@@ -47,4 +47,6 @@ CheckOptions:
     value:           lower_case
   - key:             modernize-use-emplace.SmartPointers
     value:           '::std::shared_ptr;::duckdb::unique_ptr;::std::auto_ptr;::std::weak_ptr'
+  - key:             cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams
+    value:           true
 

.github/config/out_of_tree_extensions.cmake

@@ -38,7 +38,7 @@ if (NOT MINGW)
     duckdb_extension_load(azure
             LOAD_TESTS
             GIT_URL https://github.com/duckdb/duckdb_azure
-            GIT_TAG 86f39d76157de970d16d6d6537bc90c0ee1c7d35
+            GIT_TAG 6620a32454c1eb2e455104d87262061d2464aad0
             APPLY_PATCHES
             )
 endif()

.github/config/uncovered_files.csv

@@ -1,5 +1,6 @@
 catalog/catalog.cpp	49
 catalog/catalog_entry.cpp	11
+catalog/catalog_entry_retriever.cpp	14
 catalog/catalog_entry/duck_schema_entry.cpp	10
 catalog/catalog_entry/duck_table_entry.cpp	7
 catalog/catalog_entry/index_catalog_entry.cpp	2
@@ -12,7 +13,8 @@ catalog/catalog_search_path.cpp	9
 catalog/catalog_set.cpp	20
 catalog/catalog_transaction.cpp	3
 catalog/default/default_functions.cpp	9
-catalog/dependency_manager.cpp	5
+catalog/dependency_manager.cpp	6
+catalog/dependency_list.cpp	4
 common/allocator.cpp	20
 common/arrow/arrow_appender.cpp	23
 common/arrow/appender/map_data.cpp	7
@@ -213,7 +215,7 @@ execution/operator/csv_scanner/parallel_csv_reader.cpp	54
 execution/operator/persistent/physical_batch_copy_to_file.cpp	7
 execution/operator/persistent/physical_batch_insert.cpp	43
 execution/operator/persistent/physical_copy_to_file.cpp	2
-execution/operator/persistent/physical_export.cpp	2
+execution/operator/persistent/physical_export.cpp	53
 execution/operator/persistent/physical_fixed_batch_copy.cpp	7
 execution/operator/persistent/physical_insert.cpp	5
 execution/operator/projection/physical_projection.cpp	16
@@ -342,6 +344,7 @@ extension/parquet/parquet_timestamp.cpp	12
 extension/parquet/parquet_writer.cpp	9
 extension/parquet/zstd_file_system.cpp	6
 include/duckdb/catalog/catalog.hpp	3
+include/duckdb/catalog/dependency_manager.hpp	2
 include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp	3
 include/duckdb/catalog/catalog_set.hpp	2
 include/duckdb/catalog/mapping_value.hpp	2
@@ -686,13 +689,13 @@ planner/binder/statement/bind_copy.cpp	3
 planner/binder/statement/bind_create.cpp	21
 planner/binder/statement/bind_create_table.cpp	2
 planner/binder/statement/bind_drop.cpp	20
-planner/binder/statement/bind_export.cpp	3
+planner/binder/statement/bind_export.cpp	17
 planner/binder/statement/bind_insert.cpp	19
 planner/binder/statement/bind_logical_plan.cpp	21
 planner/binder/statement/bind_pragma.cpp	3
 planner/binder/statement/bind_update.cpp	4
 planner/binder/statement/bind_vacuum.cpp	3
-planner/binder/tableref/bind_basetableref.cpp	15
+planner/binder/tableref/bind_basetableref.cpp	17
 planner/binder/tableref/bind_joinref.cpp	3
 planner/binder/tableref/bind_pivot.cpp	6
 planner/binder/tableref/bind_table_function.cpp	18

.github/workflows/IssuesCloseStale.yml

@@ -11,8 +11,11 @@ jobs:
         uses: actions/stale@v8
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 30 days.'
+          stale-pr-message: 'This pull request is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 30 days.'
           close-issue-message: 'This issue was closed because it has been stale for 30 days with no activity.'
+          close-pr-message: 'This pull request was closed because it has been stale for 30 days with no activity.'
           days-before-stale: 90
           days-before-close: 30
           operations-per-run: 500
           stale-issue-label: stale
+          stale-pr-label: stale

.github/workflows/R.yml

@@ -60,12 +60,6 @@ jobs:
           update-rtools: true
           rtools-version: '42' # linker bug in 43 ^^
 
-      - name: Setup Ccache
-        uses: hendrikmuhs/ccache-action@main
-        with:
-          key: ${{ github.job }}
-          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
-
       - uses: ./.github/actions/build_extensions
         with:
           deploy_as: windows_amd64_rtools

data/csv/quoted_newline.csv

@@ -0,0 +1,3 @@
+col1
+"cell with
+newline"

data/json/11407.json

@@ -0,0 +1,8 @@
+[
+  {
+    "k": "v"
+  },
+  {
+    "k": "v2"
+  }
+]

extension/httpfs/httpfs_extension.cpp

@@ -19,48 +19,41 @@ static void LoadInternal(DatabaseInstance &instance) {
 
 	// Global HTTP config
 	// Single timeout value is used for all 4 types of timeouts, we could split it into 4 if users need that
-	config.AddExtensionOption("http_timeout", "HTTP timeout read/write/connection/retry (default 30000ms)",
-	                          LogicalType::UBIGINT, Value(30000));
-	config.AddExtensionOption("http_retries", "HTTP retries on I/O error (default 3)", LogicalType::UBIGINT, Value(3));
-	config.AddExtensionOption("http_retry_wait_ms", "Time between retries (default 100ms)", LogicalType::UBIGINT,
-	                          Value(100));
+	config.AddExtensionOption("http_timeout", "HTTP timeout read/write/connection/retry", LogicalType::UBIGINT,
+	                          Value(30000));
+	config.AddExtensionOption("http_retries", "HTTP retries on I/O error", LogicalType::UBIGINT, Value(3));
+	config.AddExtensionOption("http_retry_wait_ms", "Time between retries", LogicalType::UBIGINT, Value(100));
 	config.AddExtensionOption("force_download", "Forces upfront download of file", LogicalType::BOOLEAN, Value(false));
 	// Reduces the number of requests made while waiting, for example retry_wait_ms of 50 and backoff factor of 2 will
 	// result in wait times of  0 50 100 200 400...etc.
-	config.AddExtensionOption("http_retry_backoff",
-	                          "Backoff factor for exponentially increasing retry wait time (default 4)",
+	config.AddExtensionOption("http_retry_backoff", "Backoff factor for exponentially increasing retry wait time",
 	                          LogicalType::FLOAT, Value(4));
 	config.AddExtensionOption(
 	    "http_keep_alive",
 	    "Keep alive connections. Setting this to false can help when running into connection failures",
 	    LogicalType::BOOLEAN, Value(true));
-	config.AddExtensionOption("enable_server_cert_verification",
-	                          "Enable server side certificate verification, defaults to False.", LogicalType::BOOLEAN,
-	                          Value(false));
-	config.AddExtensionOption("ca_cert_file",
-	                          "Path to a custom certificate file for self-signed certificates. By default not set.",
+	config.AddExtensionOption("enable_server_cert_verification", "Enable server side certificate verification.",
+	                          LogicalType::BOOLEAN, Value(false));
+	config.AddExtensionOption("ca_cert_file", "Path to a custom certificate file for self-signed certificates.",
 	                          LogicalType::VARCHAR, Value(""));
 	// Global S3 config
-	config.AddExtensionOption("s3_region", "S3 Region (default us-east-1)", LogicalType::VARCHAR, Value("us-east-1"));
+	config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR, Value("us-east-1"));
 	config.AddExtensionOption("s3_access_key_id", "S3 Access Key ID", LogicalType::VARCHAR);
 	config.AddExtensionOption("s3_secret_access_key", "S3 Access Key", LogicalType::VARCHAR);
 	config.AddExtensionOption("s3_session_token", "S3 Session Token", LogicalType::VARCHAR);
-	config.AddExtensionOption("s3_endpoint", "S3 Endpoint (empty for default endpoint)", LogicalType::VARCHAR);
-	config.AddExtensionOption("s3_url_style", "S3 URL style ('vhost' (default) or 'path')", LogicalType::VARCHAR,
-	                          Value("vhost"));
-	config.AddExtensionOption("s3_use_ssl", "S3 use SSL (default true)", LogicalType::BOOLEAN, Value(true));
+	config.AddExtensionOption("s3_endpoint", "S3 Endpoint", LogicalType::VARCHAR);
+	config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost"));
+	config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true));
 	config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs",
 	                          LogicalType::BOOLEAN, Value(false));
 
 	// S3 Uploader config
-	config.AddExtensionOption("s3_uploader_max_filesize",
-	                          "S3 Uploader max filesize (between 50GB and 5TB, default 800GB)", LogicalType::VARCHAR,
-	                          "800GB");
-	config.AddExtensionOption("s3_uploader_max_parts_per_file",
-	                          "S3 Uploader max parts per file (between 1 and 10000, default 10000)",
+	config.AddExtensionOption("s3_uploader_max_filesize", "S3 Uploader max filesize (between 50GB and 5TB)",
+	                          LogicalType::VARCHAR, "800GB");
+	config.AddExtensionOption("s3_uploader_max_parts_per_file", "S3 Uploader max parts per file (between 1 and 10000)",
 	                          LogicalType::UBIGINT, Value(10000));
-	config.AddExtensionOption("s3_uploader_thread_limit", "S3 Uploader global thread limit (default 50)",
-	                          LogicalType::UBIGINT, Value(50));
+	config.AddExtensionOption("s3_uploader_thread_limit", "S3 Uploader global thread limit", LogicalType::UBIGINT,
+	                          Value(50));
 
 	auto provider = make_uniq<AWSEnvironmentCredentialsProvider>(config);
 	provider->SetAll();

extension/json/buffered_json_reader.cpp

@@ -44,6 +44,10 @@ bool JSONFileHandle::RequestedReadsComplete() {
 	return requested_reads == actual_reads;
 }
 
+bool JSONFileHandle::LastReadRequested() const {
+	return last_read_requested;
+}
+
 idx_t JSONFileHandle::FileSize() const {
 	return file_size;
 }
@@ -56,6 +60,10 @@ bool JSONFileHandle::CanSeek() const {
 	return can_seek;
 }
 
+bool JSONFileHandle::IsPipe() const {
+	return file_handle->IsPipe();
+}
+
 FileHandle &JSONFileHandle::GetHandle() {
 	return *file_handle;
 }

extension/json/include/buffered_json_reader.hpp

@@ -60,11 +60,13 @@ struct JSONFileHandle {
 
 	void Reset();
 	bool RequestedReadsComplete();
+	bool LastReadRequested() const;
 
 	idx_t FileSize() const;
 	idx_t Remaining() const;
 
 	bool CanSeek() const;
+	bool IsPipe() const;
 
 	FileHandle &GetHandle();
 

extension/json/include/json_scan.hpp

@@ -9,13 +9,13 @@
 #pragma once
 
 #include "buffered_json_reader.hpp"
-#include "json_enums.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/types/type_map.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/function/table_function.hpp"
+#include "json_enums.hpp"
 #include "json_transform.hpp"
 
 namespace duckdb {
@@ -226,14 +226,20 @@ struct JSONScanLocalState {
 
 private:
 	bool ReadNextBuffer(JSONScanGlobalState &gstate);
-	bool ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
-	bool ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
-	bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
+	bool ReadNextBufferInternal(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                            bool &file_done);
+	bool ReadNextBufferSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                        bool &file_done);
+	bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
+	                          bool &file_done);
+	AllocatedData AllocateBuffer(JSONScanGlobalState &gstate);
+	data_ptr_t GetReconstructBuffer(JSONScanGlobalState &gstate);
+
 	void SkipOverArrayStart();
 
-	void ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index);
-	bool ReconstructFirstObject();
-	void ParseNextChunk();
+	void ReadAndAutoDetect(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index);
+	bool ReconstructFirstObject(JSONScanGlobalState &gstate);
+	void ParseNextChunk(JSONScanGlobalState &gstate);
 
 	void ParseJSON(char *const json_start, const idx_t json_size, const idx_t remaining);
 	void ThrowObjectSizeError(const idx_t object_size);

extension/json/include/json_serializer.hpp

@@ -25,15 +25,16 @@ struct JsonSerializer : Serializer {
 	void PushValue(yyjson_mut_val *val);
 
 public:
-	explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty)
+	explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty, bool skip_if_default)
 	    : doc(doc), stack({yyjson_mut_obj(doc)}), skip_if_null(skip_if_null), skip_if_empty(skip_if_empty) {
 		serialize_enum_as_string = true;
-		serialize_default_values = true;
+		serialize_default_values = !skip_if_default;
 	}
 
 	template <class T>
-	static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty) {
-		JsonSerializer serializer(doc, skip_if_null, skip_if_empty);
+	static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty,
+	                                 bool skip_if_default) {
+		JsonSerializer serializer(doc, skip_if_null, skip_if_empty, skip_if_default);
 		value.Serialize(serializer);
 		return serializer.GetRootObject();
 	}

extension/json/json_functions/json_serialize_plan.cpp

@@ -19,16 +19,19 @@ namespace duckdb {
 struct JsonSerializePlanBindData : public FunctionData {
 	bool skip_if_null = false;
 	bool skip_if_empty = false;
+	bool skip_if_default = false;
 	bool format = false;
 	bool optimize = false;
 
-	JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool format_p, bool optimize_p)
-	    : skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), format(format_p), optimize(optimize_p) {
+	JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool skip_if_default_p, bool format_p,
+	                          bool optimize_p)
+	    : skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), skip_if_default(skip_if_default_p),
+	      format(format_p), optimize(optimize_p) {
 	}
 
 public:
 	unique_ptr<FunctionData> Copy() const override {
-		return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
+		return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
 	}
 	bool Equals(const FunctionData &other_p) const override {
 		return true;
@@ -48,6 +51,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 	// Optional arguments
 	bool skip_if_null = false;
 	bool skip_if_empty = false;
+	bool skip_if_default = false;
 	bool format = false;
 	bool optimize = false;
 
@@ -69,6 +73,11 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 				throw BinderException("json_serialize_plan: 'skip_empty' argument must be a boolean");
 			}
 			skip_if_empty = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
+		} else if (arg->alias == "skip_default") {
+			if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
+				throw BinderException("json_serialize_plan: 'skip_default' argument must be a boolean");
+			}
+			skip_if_default = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
 		} else if (arg->alias == "format") {
 			if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
 				throw BinderException("json_serialize_plan: 'format' argument must be a boolean");
@@ -83,7 +92,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
 			throw BinderException(StringUtil::Format("json_serialize_plan: Unknown argument '%s'", arg->alias.c_str()));
 		}
 	}
-	return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
+	return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
 }
 
 static bool OperatorSupportsSerialization(LogicalOperator &op, string &operator_name) {
@@ -144,7 +153,8 @@ static void JsonSerializePlanFunction(DataChunk &args, ExpressionState &state, V
 					throw InvalidInputException("Operator '%s' does not support serialization", operator_name);
 				}
 
-				auto plan_json = JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty);
+				auto plan_json =
+				    JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty, info.skip_if_default);
 				yyjson_mut_arr_append(plans_arr, plan_json);
 			}